diff --git a/.gitignore b/.gitignore index a5d69d094c8..194862b3114 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml patchprocess/ +hadoop-tools/hadoop-aliyun/src/test/resources/auth-keys.xml +hadoop-tools/hadoop-aliyun/src/test/resources/contract-test-options.xml diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java index c6d188170c6..07c2a312363 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java @@ -343,8 +343,6 @@ public class KerberosAuthenticationHandler implements AuthenticationHandler { authorization = authorization.substring(KerberosAuthenticator.NEGOTIATE.length()).trim(); final Base64 base64 = new Base64(0); final byte[] clientToken = base64.decode(authorization); - final String serverName = InetAddress.getByName(request.getServerName()) - .getCanonicalHostName(); try { token = Subject.doAs(serverSubject, new PrivilegedExceptionAction() { @@ -354,10 +352,7 @@ public class KerberosAuthenticationHandler implements AuthenticationHandler { GSSContext gssContext = null; GSSCredential gssCreds = null; try { - gssCreds = gssManager.createCredential( - gssManager.createName( - KerberosUtil.getServicePrincipal("HTTP", serverName), - KerberosUtil.getOidInstance("NT_GSS_KRB5_PRINCIPAL")), + gssCreds = gssManager.createCredential(null, GSSCredential.INDEFINITE_LIFETIME, new Oid[]{ KerberosUtil.getOidInstance("GSS_SPNEGO_MECH_OID"), diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java index fda5572cdf4..66b2fde71ae 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java @@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory; public abstract class RolloverSignerSecretProvider extends SignerSecretProvider { - private static Logger LOG = LoggerFactory.getLogger( + static Logger LOG = LoggerFactory.getLogger( RolloverSignerSecretProvider.class); /** * Stores the currently valid secrets. The current secret is the 0th element diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java index 821131434db..5e640bb6e0c 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java @@ -17,7 +17,12 @@ import java.util.Arrays; import java.util.Properties; import java.util.Random; import javax.servlet.ServletContext; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.curator.test.TestingServer; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -25,7 +30,6 @@ import org.junit.Test; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.timeout; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -34,9 +38,14 @@ public class TestZKSignerSecretProvider { private TestingServer zkServer; // rollover every 2 sec - private final int timeout = 4000; + private final int timeout = 100; private final long rolloverFrequency = timeout / 2; + static final Log LOG = LogFactory.getLog(TestZKSignerSecretProvider.class); + { + LogManager.getLogger( RolloverSignerSecretProvider.LOG.getName() ).setLevel(Level.DEBUG); + } + @Before public void setup() throws Exception { zkServer = new TestingServer(); @@ -60,8 +69,8 @@ public class TestZKSignerSecretProvider { byte[] secret2 = Long.toString(rand.nextLong()).getBytes(); byte[] secret1 = Long.toString(rand.nextLong()).getBytes(); byte[] secret3 = Long.toString(rand.nextLong()).getBytes(); - ZKSignerSecretProvider secretProvider = - spy(new ZKSignerSecretProvider(seed)); + MockZKSignerSecretProvider secretProvider = + spy(new MockZKSignerSecretProvider(seed)); Properties config = new Properties(); config.setProperty( ZKSignerSecretProvider.ZOOKEEPER_CONNECTION_STRING, @@ -77,7 +86,8 @@ public class TestZKSignerSecretProvider { Assert.assertEquals(2, allSecrets.length); Assert.assertArrayEquals(secret1, allSecrets[0]); Assert.assertNull(allSecrets[1]); - verify(secretProvider, timeout(timeout).times(1)).rollSecret(); + verify(secretProvider, timeout(timeout).atLeastOnce()).rollSecret(); + secretProvider.realRollSecret(); currentSecret = secretProvider.getCurrentSecret(); allSecrets = secretProvider.getAllSecrets(); @@ -85,7 +95,8 @@ public class TestZKSignerSecretProvider { Assert.assertEquals(2, allSecrets.length); Assert.assertArrayEquals(secret2, allSecrets[0]); Assert.assertArrayEquals(secret1, allSecrets[1]); - verify(secretProvider, timeout(timeout).times(2)).rollSecret(); + verify(secretProvider, timeout(timeout).atLeast(2)).rollSecret(); + secretProvider.realRollSecret(); currentSecret = secretProvider.getCurrentSecret(); allSecrets = secretProvider.getAllSecrets(); @@ -93,35 +104,70 @@ public class TestZKSignerSecretProvider { Assert.assertEquals(2, allSecrets.length); Assert.assertArrayEquals(secret3, allSecrets[0]); Assert.assertArrayEquals(secret2, allSecrets[1]); - verify(secretProvider, timeout(timeout).times(3)).rollSecret(); + verify(secretProvider, timeout(timeout).atLeast(3)).rollSecret(); + secretProvider.realRollSecret(); } finally { secretProvider.destroy(); } } + /** + * A hack to test ZKSignerSecretProvider. + * We want to test that ZKSignerSecretProvider.rollSecret() is periodically + * called at the expected frequency, but we want to exclude the + * race-condition. + */ + private class MockZKSignerSecretProvider extends ZKSignerSecretProvider { + MockZKSignerSecretProvider(long seed) { + super(seed); + } + @Override + protected synchronized void rollSecret() { + // this is a no-op: simply used for Mockito to verify that rollSecret() + // is periodically called at the expected frequency + } + + public void realRollSecret() { + // the test code manually calls ZKSignerSecretProvider.rollSecret() + // to update the state + super.rollSecret(); + } + } + @Test - public void testMultipleInit() throws Exception { - // use the same seed so we can predict the RNG + public void testMultiple1() throws Exception { + testMultiple(1); + } + + @Test + public void testMultiple2() throws Exception { + testMultiple(2); + } + + /** + * @param order: + * 1: secretProviderA wins both realRollSecret races + * 2: secretProviderA wins 1st race, B wins 2nd + * @throws Exception + */ + public void testMultiple(int order) throws Exception { long seedA = System.currentTimeMillis(); Random rand = new Random(seedA); byte[] secretA2 = Long.toString(rand.nextLong()).getBytes(); byte[] secretA1 = Long.toString(rand.nextLong()).getBytes(); + byte[] secretA3 = Long.toString(rand.nextLong()).getBytes(); + byte[] secretA4 = Long.toString(rand.nextLong()).getBytes(); // use the same seed so we can predict the RNG long seedB = System.currentTimeMillis() + rand.nextLong(); rand = new Random(seedB); byte[] secretB2 = Long.toString(rand.nextLong()).getBytes(); byte[] secretB1 = Long.toString(rand.nextLong()).getBytes(); - // use the same seed so we can predict the RNG - long seedC = System.currentTimeMillis() + rand.nextLong(); - rand = new Random(seedC); - byte[] secretC2 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretC1 = Long.toString(rand.nextLong()).getBytes(); - ZKSignerSecretProvider secretProviderA = - spy(new ZKSignerSecretProvider(seedA)); - ZKSignerSecretProvider secretProviderB = - spy(new ZKSignerSecretProvider(seedB)); - ZKSignerSecretProvider secretProviderC = - spy(new ZKSignerSecretProvider(seedC)); + byte[] secretB3 = Long.toString(rand.nextLong()).getBytes(); + byte[] secretB4 = Long.toString(rand.nextLong()).getBytes(); + MockZKSignerSecretProvider secretProviderA = + spy(new MockZKSignerSecretProvider(seedA)); + MockZKSignerSecretProvider secretProviderB = + spy(new MockZKSignerSecretProvider(seedB)); Properties config = new Properties(); config.setProperty( ZKSignerSecretProvider.ZOOKEEPER_CONNECTION_STRING, @@ -131,106 +177,23 @@ public class TestZKSignerSecretProvider { try { secretProviderA.init(config, getDummyServletContext(), rolloverFrequency); secretProviderB.init(config, getDummyServletContext(), rolloverFrequency); - secretProviderC.init(config, getDummyServletContext(), rolloverFrequency); byte[] currentSecretA = secretProviderA.getCurrentSecret(); byte[][] allSecretsA = secretProviderA.getAllSecrets(); byte[] currentSecretB = secretProviderB.getCurrentSecret(); byte[][] allSecretsB = secretProviderB.getAllSecrets(); - byte[] currentSecretC = secretProviderC.getCurrentSecret(); - byte[][] allSecretsC = secretProviderC.getAllSecrets(); - Assert.assertArrayEquals(currentSecretA, currentSecretB); - Assert.assertArrayEquals(currentSecretB, currentSecretC); + Assert.assertArrayEquals(secretA1, currentSecretA); + Assert.assertArrayEquals(secretA1, currentSecretB); Assert.assertEquals(2, allSecretsA.length); Assert.assertEquals(2, allSecretsB.length); - Assert.assertEquals(2, allSecretsC.length); - Assert.assertArrayEquals(allSecretsA[0], allSecretsB[0]); - Assert.assertArrayEquals(allSecretsB[0], allSecretsC[0]); + Assert.assertArrayEquals(secretA1, allSecretsA[0]); + Assert.assertArrayEquals(secretA1, allSecretsB[0]); Assert.assertNull(allSecretsA[1]); Assert.assertNull(allSecretsB[1]); - Assert.assertNull(allSecretsC[1]); - char secretChosen = 'z'; - if (Arrays.equals(secretA1, currentSecretA)) { - Assert.assertArrayEquals(secretA1, allSecretsA[0]); - secretChosen = 'A'; - } else if (Arrays.equals(secretB1, currentSecretB)) { - Assert.assertArrayEquals(secretB1, allSecretsA[0]); - secretChosen = 'B'; - }else if (Arrays.equals(secretC1, currentSecretC)) { - Assert.assertArrayEquals(secretC1, allSecretsA[0]); - secretChosen = 'C'; - } else { - Assert.fail("It appears that they all agreed on the same secret, but " - + "not one of the secrets they were supposed to"); - } - verify(secretProviderA, timeout(timeout).times(1)).rollSecret(); - verify(secretProviderB, timeout(timeout).times(1)).rollSecret(); - verify(secretProviderC, timeout(timeout).times(1)).rollSecret(); - - currentSecretA = secretProviderA.getCurrentSecret(); - allSecretsA = secretProviderA.getAllSecrets(); - currentSecretB = secretProviderB.getCurrentSecret(); - allSecretsB = secretProviderB.getAllSecrets(); - currentSecretC = secretProviderC.getCurrentSecret(); - allSecretsC = secretProviderC.getAllSecrets(); - Assert.assertArrayEquals(currentSecretA, currentSecretB); - Assert.assertArrayEquals(currentSecretB, currentSecretC); - Assert.assertEquals(2, allSecretsA.length); - Assert.assertEquals(2, allSecretsB.length); - Assert.assertEquals(2, allSecretsC.length); - Assert.assertArrayEquals(allSecretsA[0], allSecretsB[0]); - Assert.assertArrayEquals(allSecretsB[0], allSecretsC[0]); - Assert.assertArrayEquals(allSecretsA[1], allSecretsB[1]); - Assert.assertArrayEquals(allSecretsB[1], allSecretsC[1]); - // The second secret used is prechosen by whoever won the init; so it - // should match with whichever we saw before - if (secretChosen == 'A') { - Assert.assertArrayEquals(secretA2, currentSecretA); - } else if (secretChosen == 'B') { - Assert.assertArrayEquals(secretB2, currentSecretA); - } else if (secretChosen == 'C') { - Assert.assertArrayEquals(secretC2, currentSecretA); - } - } finally { - secretProviderC.destroy(); - secretProviderB.destroy(); - secretProviderA.destroy(); - } - } - - @Test - public void testMultipleUnsychnronized() throws Exception { - long seedA = System.currentTimeMillis(); - Random rand = new Random(seedA); - byte[] secretA2 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretA1 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretA3 = Long.toString(rand.nextLong()).getBytes(); - // use the same seed so we can predict the RNG - long seedB = System.currentTimeMillis() + rand.nextLong(); - rand = new Random(seedB); - byte[] secretB2 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretB1 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretB3 = Long.toString(rand.nextLong()).getBytes(); - ZKSignerSecretProvider secretProviderA = - spy(new ZKSignerSecretProvider(seedA)); - ZKSignerSecretProvider secretProviderB = - spy(new ZKSignerSecretProvider(seedB)); - Properties config = new Properties(); - config.setProperty( - ZKSignerSecretProvider.ZOOKEEPER_CONNECTION_STRING, - zkServer.getConnectString()); - config.setProperty(ZKSignerSecretProvider.ZOOKEEPER_PATH, - "/secret"); - try { - secretProviderA.init(config, getDummyServletContext(), rolloverFrequency); - - byte[] currentSecretA = secretProviderA.getCurrentSecret(); - byte[][] allSecretsA = secretProviderA.getAllSecrets(); - Assert.assertArrayEquals(secretA1, currentSecretA); - Assert.assertEquals(2, allSecretsA.length); - Assert.assertArrayEquals(secretA1, allSecretsA[0]); - Assert.assertNull(allSecretsA[1]); - verify(secretProviderA, timeout(timeout).times(1)).rollSecret(); + verify(secretProviderA, timeout(timeout).atLeastOnce()).rollSecret(); + verify(secretProviderB, timeout(timeout).atLeastOnce()).rollSecret(); + secretProviderA.realRollSecret(); + secretProviderB.realRollSecret(); currentSecretA = secretProviderA.getCurrentSecret(); allSecretsA = secretProviderA.getAllSecrets(); @@ -238,18 +201,32 @@ public class TestZKSignerSecretProvider { Assert.assertEquals(2, allSecretsA.length); Assert.assertArrayEquals(secretA2, allSecretsA[0]); Assert.assertArrayEquals(secretA1, allSecretsA[1]); - Thread.sleep((rolloverFrequency / 5)); - secretProviderB.init(config, getDummyServletContext(), rolloverFrequency); - - byte[] currentSecretB = secretProviderB.getCurrentSecret(); - byte[][] allSecretsB = secretProviderB.getAllSecrets(); + currentSecretB = secretProviderB.getCurrentSecret(); + allSecretsB = secretProviderB.getAllSecrets(); Assert.assertArrayEquals(secretA2, currentSecretB); Assert.assertEquals(2, allSecretsA.length); Assert.assertArrayEquals(secretA2, allSecretsB[0]); Assert.assertArrayEquals(secretA1, allSecretsB[1]); - verify(secretProviderA, timeout(timeout).times(2)).rollSecret(); - verify(secretProviderB, timeout(timeout).times(1)).rollSecret(); + verify(secretProviderA, timeout(timeout).atLeast(2)).rollSecret(); + verify(secretProviderB, timeout(timeout).atLeastOnce()).rollSecret(); + + switch (order) { + case 1: + secretProviderA.realRollSecret(); + secretProviderB.realRollSecret(); + secretProviderA.realRollSecret(); + secretProviderB.realRollSecret(); + break; + case 2: + secretProviderB.realRollSecret(); + secretProviderA.realRollSecret(); + secretProviderB.realRollSecret(); + secretProviderA.realRollSecret(); + break; + default: + throw new Exception("Invalid order selected"); + } currentSecretA = secretProviderA.getCurrentSecret(); allSecretsA = secretProviderA.getAllSecrets(); @@ -260,13 +237,13 @@ public class TestZKSignerSecretProvider { Assert.assertEquals(2, allSecretsB.length); Assert.assertArrayEquals(allSecretsA[0], allSecretsB[0]); Assert.assertArrayEquals(allSecretsA[1], allSecretsB[1]); - if (Arrays.equals(secretA3, currentSecretA)) { - Assert.assertArrayEquals(secretA3, allSecretsA[0]); - } else if (Arrays.equals(secretB3, currentSecretB)) { - Assert.assertArrayEquals(secretB3, allSecretsA[0]); - } else { - Assert.fail("It appears that they all agreed on the same secret, but " - + "not one of the secrets they were supposed to"); + switch (order) { + case 1: + Assert.assertArrayEquals(secretA4, allSecretsA[0]); + break; + case 2: + Assert.assertArrayEquals(secretB4, allSecretsA[0]); + break; } } finally { secretProviderB.destroy(); diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml index ec7c3967af4..bded4b99514 100644 --- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml +++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml @@ -405,4 +405,9 @@ + + + + + diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 54d1cddb8a4..92582aede6a 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -235,6 +235,7 @@ com.jcraft jsch + provided org.apache.curator diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java index 700487118d9..cdc95813087 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java @@ -70,11 +70,14 @@ public class ConfServlet extends HttpServlet { response.setContentType("application/json; charset=utf-8"); } + String name = request.getParameter("name"); Writer out = response.getWriter(); try { - writeResponse(getConfFromContext(), out, format); + writeResponse(getConfFromContext(), out, format, name); } catch (BadFormatException bfe) { response.sendError(HttpServletResponse.SC_BAD_REQUEST, bfe.getMessage()); + } catch (IllegalArgumentException iae) { + response.sendError(HttpServletResponse.SC_NOT_FOUND, iae.getMessage()); } out.close(); } @@ -89,17 +92,23 @@ public class ConfServlet extends HttpServlet { /** * Guts of the servlet - extracted for easy testing. */ - static void writeResponse(Configuration conf, Writer out, String format) - throws IOException, BadFormatException { + static void writeResponse(Configuration conf, + Writer out, String format, String propertyName) + throws IOException, IllegalArgumentException, BadFormatException { if (FORMAT_JSON.equals(format)) { - Configuration.dumpConfiguration(conf, out); + Configuration.dumpConfiguration(conf, propertyName, out); } else if (FORMAT_XML.equals(format)) { - conf.writeXml(out); + conf.writeXml(propertyName, out); } else { throw new BadFormatException("Bad format: " + format); } } + static void writeResponse(Configuration conf, Writer out, String format) + throws IOException, BadFormatException { + writeResponse(conf, out, format, null); + } + public static class BadFormatException extends Exception { private static final long serialVersionUID = 1L; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 66734b6519f..dbbc8ff20e8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -103,8 +103,9 @@ import org.w3c.dom.Text; import org.xml.sax.SAXException; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; -/** +/** * Provides access to configuration parameters. * *

Resources

@@ -942,10 +943,15 @@ public class Configuration implements Iterable>, * * If var is unbounded the current state of expansion "prefix${var}suffix" is * returned. - * - * If a cycle is detected: replacing var1 requires replacing var2 ... requires - * replacing var1, i.e., the cycle is shorter than - * {@link Configuration#MAX_SUBST} then the original expr is returned. + *

+ * This function also detects self-referential substitutions, i.e. + * 

+   *   {@code
+   *   foo.bar = ${foo.bar}
+   *   }
+   *
+ * If a cycle is detected then the original expr is returned. Loops + * involving multiple substitutions are not detected. * * @param expr the literal value of a config key * @return null if expr is null, otherwise the value resulting from expanding @@ -958,7 +964,6 @@ public class Configuration implements Iterable>, return null; } String eval = expr; - Set evalSet = null; for(int s = 0; s < MAX_SUBST; s++) { final int[] varBounds = findSubVariable(eval); if (varBounds[SUB_START_IDX] == -1) { @@ -1003,15 +1008,12 @@ public class Configuration implements Iterable>, return eval; // return literal ${var}: var is unbound } - // prevent recursive resolution - // final int dollar = varBounds[SUB_START_IDX] - "${".length(); final int afterRightBrace = varBounds[SUB_END_IDX] + "}".length(); final String refVar = eval.substring(dollar, afterRightBrace); - if (evalSet == null) { - evalSet = new HashSet(); - } - if (!evalSet.add(refVar)) { + + // detect self-referential values + if (val.contains(refVar)) { return expr; // return original expression if there is a loop } @@ -2834,14 +2836,37 @@ public class Configuration implements Iterable>, writeXml(new OutputStreamWriter(out, "UTF-8")); } - /** - * Write out the non-default properties in this configuration to the given - * {@link Writer}. - * + public void writeXml(Writer out) throws IOException { + writeXml(null, out); + } + + /** + * Write out the non-default properties in this configuration to the + * given {@link Writer}. + * + *
  • + * When property name is not empty and the property exists in the + * configuration, this method writes the property and its attributes + * to the {@link Writer}. + *
    • + *

    + * + *

  • + * When property name is null or empty, this method writes all the + * configuration properties and their attributes to the {@link Writer}. + *
    • + *

    + * + *

  • + * When property name is not empty but the property doesn't exist in + * the configuration, this method throws an {@link IllegalArgumentException}. + *
    • + *

    * @param out the writer to write to. */ - public void writeXml(Writer out) throws IOException { - Document doc = asXmlDocument(); + public void writeXml(String propertyName, Writer out) + throws IOException, IllegalArgumentException { + Document doc = asXmlDocument(propertyName); try { DOMSource source = new DOMSource(doc); @@ -2861,62 +2886,180 @@ public class Configuration implements Iterable>, /** * Return the XML DOM corresponding to this Configuration. */ - private synchronized Document asXmlDocument() throws IOException { + private synchronized Document asXmlDocument(String propertyName) + throws IOException, IllegalArgumentException { Document doc; try { - doc = - DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); + doc = DocumentBuilderFactory + .newInstance() + .newDocumentBuilder() + .newDocument(); } catch (ParserConfigurationException pe) { throw new IOException(pe); } + Element conf = doc.createElement("configuration"); doc.appendChild(conf); conf.appendChild(doc.createTextNode("\n")); handleDeprecation(); //ensure properties is set and deprecation is handled - for (Enumeration e = properties.keys(); e.hasMoreElements();) { - String name = (String)e.nextElement(); - Object object = properties.get(name); - String value = null; - if (object instanceof String) { - value = (String) object; - }else { - continue; + + if(!Strings.isNullOrEmpty(propertyName)) { + if (!properties.containsKey(propertyName)) { + // given property not found, illegal argument + throw new IllegalArgumentException("Property " + + propertyName + " not found"); + } else { + // given property is found, write single property + appendXMLProperty(doc, conf, propertyName); + conf.appendChild(doc.createTextNode("\n")); } - Element propNode = doc.createElement("property"); - conf.appendChild(propNode); - - Element nameNode = doc.createElement("name"); - nameNode.appendChild(doc.createTextNode(name)); - propNode.appendChild(nameNode); - - Element valueNode = doc.createElement("value"); - valueNode.appendChild(doc.createTextNode(value)); - propNode.appendChild(valueNode); - - if (updatingResource != null) { - String[] sources = updatingResource.get(name); - if(sources != null) { - for(String s : sources) { - Element sourceNode = doc.createElement("source"); - sourceNode.appendChild(doc.createTextNode(s)); - propNode.appendChild(sourceNode); - } - } + } else { + // append all elements + for (Enumeration e = properties.keys(); e.hasMoreElements();) { + appendXMLProperty(doc, conf, (String)e.nextElement()); + conf.appendChild(doc.createTextNode("\n")); } - - conf.appendChild(doc.createTextNode("\n")); } return doc; } /** - * Writes out all the parameters and their properties (final and resource) to - * the given {@link Writer} - * The format of the output would be - * { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2, - * key2.isFinal,key2.resource}... ] } - * It does not output the parameters of the configuration object which is - * loaded from an input stream. + * Append a property with its attributes to a given {#link Document} + * if the property is found in configuration. + * + * @param doc + * @param conf + * @param propertyName + */ + private synchronized void appendXMLProperty(Document doc, Element conf, + String propertyName) { + // skip writing if given property name is empty or null + if (!Strings.isNullOrEmpty(propertyName)) { + String value = properties.getProperty(propertyName); + if (value != null) { + Element propNode = doc.createElement("property"); + conf.appendChild(propNode); + + Element nameNode = doc.createElement("name"); + nameNode.appendChild(doc.createTextNode(propertyName)); + propNode.appendChild(nameNode); + + Element valueNode = doc.createElement("value"); + valueNode.appendChild(doc.createTextNode( + properties.getProperty(propertyName))); + propNode.appendChild(valueNode); + + Element finalNode = doc.createElement("final"); + finalNode.appendChild(doc.createTextNode( + String.valueOf(finalParameters.contains(propertyName)))); + propNode.appendChild(finalNode); + + if (updatingResource != null) { + String[] sources = updatingResource.get(propertyName); + if(sources != null) { + for(String s : sources) { + Element sourceNode = doc.createElement("source"); + sourceNode.appendChild(doc.createTextNode(s)); + propNode.appendChild(sourceNode); + } + } + } + } + } + } + + /** + * Writes properties and their attributes (final and resource) + * to the given {@link Writer}. + * + *
  • + * When propertyName is not empty, and the property exists + * in the configuration, the format of the output would be, + * 
    +   *  {
+   *    "property": {
+   *      "key" : "key1",
+   *      "value" : "value1",
+   *      "isFinal" : "key1.isFinal",
+   *      "resource" : "key1.resource"
+   *    }
+   *  }
+   *
    + *
    • + * + *
  • + * When propertyName is null or empty, it behaves same as + * {@link #dumpConfiguration(Configuration, Writer)}, the + * output would be, + * 
    +   *  { "properties" :
+   *      [ { key : "key1",
+   *          value : "value1",
+   *          isFinal : "key1.isFinal",
+   *          resource : "key1.resource" },
+   *        { key : "key2",
+   *          value : "value2",
+   *          isFinal : "ke2.isFinal",
+   *          resource : "key2.resource" }
+   *       ]
+   *   }
+   *
    + *
    • + * + *
  • + * When propertyName is not empty, and the property is not + * found in the configuration, this method will throw an + * {@link IllegalArgumentException}. + *
    • + *

    + * @param config the configuration + * @param propertyName property name + * @param out the Writer to write to + * @throws IOException + * @throws IllegalArgumentException when property name is not + * empty and the property is not found in configuration + **/ + public static void dumpConfiguration(Configuration config, + String propertyName, Writer out) throws IOException { + if(Strings.isNullOrEmpty(propertyName)) { + dumpConfiguration(config, out); + } else if (Strings.isNullOrEmpty(config.get(propertyName))) { + throw new IllegalArgumentException("Property " + + propertyName + " not found"); + } else { + JsonFactory dumpFactory = new JsonFactory(); + JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out); + dumpGenerator.writeStartObject(); + dumpGenerator.writeFieldName("property"); + appendJSONProperty(dumpGenerator, config, propertyName); + dumpGenerator.writeEndObject(); + dumpGenerator.flush(); + } + } + + /** + * Writes out all properties and their attributes (final and resource) to + * the given {@link Writer}, the format of the output would be, + * + * 

    +   *  { "properties" :
+   *      [ { key : "key1",
+   *          value : "value1",
+   *          isFinal : "key1.isFinal",
+   *          resource : "key1.resource" },
+   *        { key : "key2",
+   *          value : "value2",
+   *          isFinal : "ke2.isFinal",
+   *          resource : "key2.resource" }
+   *       ]
+   *   }
+   *
    + * + * It does not output the properties of the configuration object which + * is loaded from an input stream. + *

    + * + * @param config the configuration * @param out the Writer to write to * @throws IOException */ @@ -2930,29 +3073,47 @@ public class Configuration implements Iterable>, dumpGenerator.flush(); synchronized (config) { for (Map.Entry item: config.getProps().entrySet()) { - dumpGenerator.writeStartObject(); - dumpGenerator.writeStringField("key", (String) item.getKey()); - dumpGenerator.writeStringField("value", - config.get((String) item.getKey())); - dumpGenerator.writeBooleanField("isFinal", - config.finalParameters.contains(item.getKey())); - String[] resources = config.updatingResource.get(item.getKey()); - String resource = UNKNOWN_RESOURCE; - if(resources != null && resources.length > 0) { - resource = resources[0]; - } - dumpGenerator.writeStringField("resource", resource); - dumpGenerator.writeEndObject(); + appendJSONProperty(dumpGenerator, + config, + item.getKey().toString()); } } dumpGenerator.writeEndArray(); dumpGenerator.writeEndObject(); dumpGenerator.flush(); } - + + /** + * Write property and its attributes as json format to given + * {@link JsonGenerator}. + * + * @param jsonGen json writer + * @param config configuration + * @param name property name + * @throws IOException + */ + private static void appendJSONProperty(JsonGenerator jsonGen, + Configuration config, String name) throws IOException { + // skip writing if given property name is empty or null + if(!Strings.isNullOrEmpty(name) && jsonGen != null) { + jsonGen.writeStartObject(); + jsonGen.writeStringField("key", name); + jsonGen.writeStringField("value", config.get(name)); + jsonGen.writeBooleanField("isFinal", + config.finalParameters.contains(name)); + String[] resources = config.updatingResource.get(name); + String resource = UNKNOWN_RESOURCE; + if(resources != null && resources.length > 0) { + resource = resources[0]; + } + jsonGen.writeStringField("resource", resource); + jsonGen.writeEndObject(); + } + } + /** * Get the {@link ClassLoader} for this job. - * + * * @return the correct class loader. */ public ClassLoader getClassLoader() { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java index 505f76dd81d..a2b6980aa89 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java @@ -177,7 +177,8 @@ public abstract class CachingGetSpaceUsed implements Closeable, GetSpaceUsed { // update the used variable spaceUsed.refresh(); } catch (InterruptedException e) { - LOG.warn("Thread Interrupted waiting to refresh disk information", e); + LOG.warn("Thread Interrupted waiting to refresh disk information: " + + e.getMessage()); Thread.currentThread().interrupt(); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DFCachingGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DFCachingGetSpaceUsed.java new file mode 100644 index 00000000000..6e8cd461eea --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DFCachingGetSpaceUsed.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import java.io.IOException; + +/** + * Fast but inaccurate class to tell how much space HDFS is using. + * This class makes the assumption that the entire mount is used for + * HDFS and that no two hdfs data dirs are on the same disk. + * + * To use set fs.getspaceused.classname + * to org.apache.hadoop.fs.DFCachingGetSpaceUsed in your core-site.xml + * + */ +@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) +@InterfaceStability.Evolving +public class DFCachingGetSpaceUsed extends CachingGetSpaceUsed { + private final DF df; + + public DFCachingGetSpaceUsed(Builder builder) throws IOException { + super(builder); + this.df = new DF(builder.getPath(), builder.getInterval()); + } + + @Override + protected void refresh() { + this.used.set(df.getUsed()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java index 20e8202ea2e..b64a19d3cf0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java @@ -31,12 +31,13 @@ import java.io.IOException; @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Evolving public class DU extends CachingGetSpaceUsed { - private DUShell duShell; + private final DUShell duShell; @VisibleForTesting - public DU(File path, long interval, long jitter, long initialUsed) + public DU(File path, long interval, long jitter, long initialUsed) throws IOException { super(path, interval, jitter, initialUsed); + this.duShell = new DUShell(); } public DU(CachingGetSpaceUsed.Builder builder) throws IOException { @@ -48,9 +49,6 @@ public class DU extends CachingGetSpaceUsed { @Override protected synchronized void refresh() { - if (duShell == null) { - duShell = new DUShell(); - } try { duShell.startRefresh(); } catch (IOException ioe) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java index 00ddfe8fff1..1129e077fde 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java @@ -121,4 +121,25 @@ public class FileEncryptionInfo { builder.append("}"); return builder.toString(); } + + /** + * A frozen version of {@link #toString()} to be backward compatible. + * When backward compatibility is not needed, use {@link #toString()}, which + * provides more info and is supposed to evolve. + * Don't change this method except for major revisions. + * + * NOTE: + * Currently this method is used by CLI for backward compatibility. + */ + public String toStringStable() { + StringBuilder builder = new StringBuilder("{"); + builder.append("cipherSuite: " + cipherSuite); + builder.append(", cryptoProtocolVersion: " + version); + builder.append(", edek: " + Hex.encodeHexString(edek)); + builder.append(", iv: " + Hex.encodeHexString(iv)); + builder.append(", keyName: " + keyName); + builder.append(", ezKeyVersionName: " + ezKeyVersionName); + builder.append("}"); + return builder.toString(); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index c36598f01b2..cc062c4ecae 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2858,7 +2858,15 @@ public abstract class FileSystem extends Configured implements Closeable { ClassUtil.findContainingJar(fs.getClass()), e); } } catch (ServiceConfigurationError ee) { - LOG.warn("Cannot load filesystem", ee); + LOG.warn("Cannot load filesystem: " + ee); + Throwable cause = ee.getCause(); + // print all the nested exception messages + while (cause != null) { + LOG.warn(cause.toString()); + cause = cause.getCause(); + } + // and at debug: the full stack + LOG.debug("Stack Trace", ee); } } FILE_SYSTEMS_LOADED = true; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java index bd99db403d6..2fe3fd1a877 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java @@ -36,15 +36,25 @@ public abstract class TrashPolicy extends Configured { protected Path trash; // path to trash directory protected long deletionInterval; // deletion interval for Emptier + /** + * Used to setup the trash policy. Must be implemented by all TrashPolicy + * implementations. + * @param conf the configuration to be used + * @param fs the filesystem to be used + * @param home the home directory + * @deprecated Use {@link #initialize(Configuration, FileSystem)} instead. + */ + @Deprecated + public abstract void initialize(Configuration conf, FileSystem fs, Path home); + /** * Used to setup the trash policy. Must be implemented by all TrashPolicy * implementations. Different from initialize(conf, fs, home), this one does * not assume trash always under /user/$USER due to HDFS encryption zone. * @param conf the configuration to be used * @param fs the filesystem to be used - * @throws IOException */ - public void initialize(Configuration conf, FileSystem fs) throws IOException{ + public void initialize(Configuration conf, FileSystem fs) { throw new UnsupportedOperationException(); } @@ -99,6 +109,25 @@ public abstract class TrashPolicy extends Configured { */ public abstract Runnable getEmptier() throws IOException; + /** + * Get an instance of the configured TrashPolicy based on the value + * of the configuration parameter fs.trash.classname. + * + * @param conf the configuration to be used + * @param fs the file system to be used + * @param home the home directory + * @return an instance of TrashPolicy + * @deprecated Use {@link #getInstance(Configuration, FileSystem)} instead. + */ + @Deprecated + public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) { + Class trashClass = conf.getClass( + "fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class); + TrashPolicy trash = ReflectionUtils.newInstance(trashClass, conf); + trash.initialize(conf, fs, home); // initialize TrashPolicy + return trash; + } + /** * Get an instance of the configured TrashPolicy based on the value * of the configuration parameter fs.trash.classname. @@ -107,8 +136,7 @@ public abstract class TrashPolicy extends Configured { * @param fs the file system to be used * @return an instance of TrashPolicy */ - public static TrashPolicy getInstance(Configuration conf, FileSystem fs) - throws IOException { + public static TrashPolicy getInstance(Configuration conf, FileSystem fs) { Class trashClass = conf.getClass( "fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class); TrashPolicy trash = ReflectionUtils.newInstance(trashClass, conf); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java index f4a825c451e..72222be04a8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java @@ -75,6 +75,21 @@ public class TrashPolicyDefault extends TrashPolicy { initialize(conf, fs); } + /** + * @deprecated Use {@link #initialize(Configuration, FileSystem)} instead. + */ + @Override + @Deprecated + public void initialize(Configuration conf, FileSystem fs, Path home) { + this.fs = fs; + this.deletionInterval = (long)(conf.getFloat( + FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT) + * MSECS_PER_MINUTE); + this.emptierInterval = (long)(conf.getFloat( + FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) + * MSECS_PER_MINUTE); + } + @Override public void initialize(Configuration conf, FileSystem fs) { this.fs = fs; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java index 45402f8a2f7..b42c36525a8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java @@ -36,7 +36,7 @@ import org.apache.hadoop.util.StringUtils; * to create a new instance. */ @InterfaceAudience.Public -@InterfaceStability.Evolving +@InterfaceStability.Stable public class AclEntry { private final AclEntryType type; private final String name; @@ -100,13 +100,29 @@ public class AclEntry { } @Override + @InterfaceStability.Unstable public String toString() { + // This currently just delegates to the stable string representation, but it + // is permissible for the output of this method to change across versions. + return toStringStable(); + } + + /** + * Returns a string representation guaranteed to be stable across versions to + * satisfy backward compatibility requirements, such as for shell command + * output or serialization. The format of this string representation matches + * what is expected by the {@link #parseAclSpec(String, boolean)} and + * {@link #parseAclEntry(String, boolean)} methods. + * + * @return stable, backward compatible string representation + */ + public String toStringStable() { StringBuilder sb = new StringBuilder(); if (scope == AclEntryScope.DEFAULT) { sb.append("default:"); } if (type != null) { - sb.append(StringUtils.toLowerCase(type.toString())); + sb.append(StringUtils.toLowerCase(type.toStringStable())); } sb.append(':'); if (name != null) { @@ -203,6 +219,8 @@ public class AclEntry { /** * Parses a string representation of an ACL spec into a list of AclEntry * objects. Example: "user::rwx,user:foo:rw-,group::r--,other::---" + * The expected format of ACL entries in the string parameter is the same + * format produced by the {@link #toStringStable()} method. * * @param aclSpec * String representation of an ACL spec. @@ -228,6 +246,8 @@ public class AclEntry { /** * Parses a string representation of an ACL into a AclEntry object.
    + * The expected format of ACL entries in the string parameter is the same + * format produced by the {@link #toStringStable()} method. * * @param aclStr * String representation of an ACL.
    diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntryScope.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntryScope.java index 6d941e7117d..64c70aa8ca8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntryScope.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntryScope.java @@ -24,7 +24,7 @@ import org.apache.hadoop.classification.InterfaceStability; * Specifies the scope or intended usage of an ACL entry. */ @InterfaceAudience.Public -@InterfaceStability.Evolving +@InterfaceStability.Stable public enum AclEntryScope { /** * An ACL entry that is inspected during permission checks to enforce diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntryType.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntryType.java index ffd62d7080b..002ead2b6a9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntryType.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntryType.java @@ -24,7 +24,7 @@ import org.apache.hadoop.classification.InterfaceStability; * Specifies the type of an ACL entry. */ @InterfaceAudience.Public -@InterfaceStability.Evolving +@InterfaceStability.Stable public enum AclEntryType { /** * An ACL entry applied to a specific user. These ACL entries can be unnamed, @@ -55,4 +55,25 @@ public enum AclEntryType { * of the more specific ACL entry types. */ OTHER; + + @Override + @InterfaceStability.Unstable + public String toString() { + // This currently just delegates to the stable string representation, but it + // is permissible for the output of this method to change across versions. + return toStringStable(); + } + + /** + * Returns a string representation guaranteed to be stable across versions to + * satisfy backward compatibility requirements, such as for shell command + * output or serialization. + * + * @return stable, backward compatible string representation + */ + public String toStringStable() { + // The base implementation uses the enum value names, which are public API + // and therefore stable. + return super.toString(); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java index 9d7500a697b..131aa199435 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java @@ -31,7 +31,7 @@ import com.google.common.collect.Lists; * instances are immutable. Use a {@link Builder} to create a new instance. */ @InterfaceAudience.Public -@InterfaceStability.Evolving +@InterfaceStability.Stable public class AclStatus { private final String owner; private final String group; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java index 9a5404032b0..a5e386c785e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java @@ -117,7 +117,7 @@ class AclCommands extends FsCommand { } if (AclUtil.isMinimalAcl(entries)) { for (AclEntry entry: entries) { - out.println(entry); + out.println(entry.toStringStable()); } } else { for (AclEntry entry: entries) { @@ -145,10 +145,10 @@ class AclCommands extends FsCommand { out.println(String.format("%s\t#effective:%s", entry, effectivePerm.SYMBOL)); } else { - out.println(entry); + out.println(entry.toStringStable()); } } else { - out.println(entry); + out.println(entry.toStringStable()); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java index 2a9c5d0bc2a..20a4cd6c013 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java @@ -60,20 +60,22 @@ public class SnappyCodec implements Configurable, CompressionCodec, DirectDecomp * Are the native snappy libraries loaded & initialized? */ public static void checkNativeCodeLoaded() { - if (!NativeCodeLoader.isNativeCodeLoaded() || - !NativeCodeLoader.buildSupportsSnappy()) { - throw new RuntimeException("native snappy library not available: " + - "this version of libhadoop was built without " + - "snappy support."); - } - if (!SnappyCompressor.isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available: " + - "SnappyCompressor has not been loaded."); - } - if (!SnappyDecompressor.isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available: " + - "SnappyDecompressor has not been loaded."); - } + if (!NativeCodeLoader.buildSupportsSnappy()) { + throw new RuntimeException("native snappy library not available: " + + "this version of libhadoop was built without " + + "snappy support."); + } + if (!NativeCodeLoader.isNativeCodeLoaded()) { + throw new RuntimeException("Failed to load libhadoop."); + } + if (!SnappyCompressor.isNativeCodeLoaded()) { + throw new RuntimeException("native snappy library not available: " + + "SnappyCompressor has not been loaded."); + } + if (!SnappyDecompressor.isNativeCodeLoaded()) { + throw new RuntimeException("native snappy library not available: " + + "SnappyDecompressor has not been loaded."); + } } public static boolean isNativeCodeLoaded() { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ExternalCall.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ExternalCall.java new file mode 100644 index 00000000000..556613639bf --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ExternalCall.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ipc; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.ipc.Server.Call; +import org.apache.hadoop.security.UserGroupInformation; + +public abstract class ExternalCall extends Call { + private final PrivilegedExceptionAction action; + private final AtomicBoolean done = new AtomicBoolean(); + private T result; + private Throwable error; + + public ExternalCall(PrivilegedExceptionAction action) { + this.action = action; + } + + public abstract UserGroupInformation getRemoteUser(); + + public final T get() throws InterruptedException, ExecutionException { + waitForCompletion(); + if (error != null) { + throw new ExecutionException(error); + } + return result; + } + + // wait for response to be triggered to support postponed calls + private void waitForCompletion() throws InterruptedException { + synchronized(done) { + while (!done.get()) { + try { + done.wait(); + } catch (InterruptedException ie) { + if (Thread.interrupted()) { + throw ie; + } + } + } + } + } + + boolean isDone() { + return done.get(); + } + + // invoked by ipc handler + @Override + public final Void run() throws IOException { + try { + result = action.run(); + sendResponse(); + } catch (Throwable t) { + abortResponse(t); + } + return null; + } + + @Override + final void doResponse(Throwable t) { + synchronized(done) { + error = t; + done.set(true); + done.notify(); + } + } +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index f509d71517f..1c7e76a5068 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -384,6 +384,11 @@ public abstract class Server { return (call != null) ? call.getRemoteUser() : null; } + public static String getProtocol() { + Call call = CurCall.get(); + return (call != null) ? call.getProtocol() : null; + } + /** Return true if the invocation was through an RPC. */ public static boolean isRpcInvocation() { @@ -672,6 +677,11 @@ public abstract class Server { private int priorityLevel; // the priority level assigned by scheduler, 0 by default + Call() { + this(RpcConstants.INVALID_CALL_ID, RpcConstants.INVALID_RETRY_COUNT, + RPC.RpcKind.RPC_BUILTIN, RpcConstants.DUMMY_CLIENT_ID); + } + Call(Call call) { this(call.callId, call.retryCount, call.rpcKind, call.clientId, call.traceScope, call.callerContext); @@ -703,6 +713,7 @@ public abstract class Server { return "Call#" + callId + " Retry#" + retryCount; } + @Override public Void run() throws Exception { return null; } @@ -718,6 +729,10 @@ public abstract class Server { return (addr != null) ? addr.getHostAddress() : null; } + public String getProtocol() { + return null; + } + /** * Allow a IPC response to be postponed instead of sent immediately * after the handler returns from the proxy method. The intended use @@ -799,6 +814,11 @@ public abstract class Server { this.rpcRequest = param; } + @Override + public String getProtocol() { + return "rpc"; + } + @Override public UserGroupInformation getRemoteUser() { return connection.user; @@ -2333,33 +2353,15 @@ public abstract class Server { // Save the priority level assignment by the scheduler call.setPriorityLevel(callQueue.getPriorityLevel(call)); - if (callQueue.isClientBackoffEnabled()) { - // if RPC queue is full, we will ask the RPC client to back off by - // throwing RetriableException. Whether RPC client will honor - // RetriableException and retry depends on client ipc retry policy. - // For example, FailoverOnNetworkExceptionRetry handles - // RetriableException. - queueRequestOrAskClientToBackOff(call); - } else { - callQueue.put(call); // queue the call; maybe blocked here + try { + queueCall(call); + } catch (IOException ioe) { + throw new WrappedRpcServerException( + RpcErrorCodeProto.ERROR_RPC_SERVER, ioe); } incRpcCount(); // Increment the rpc count } - private void queueRequestOrAskClientToBackOff(Call call) - throws WrappedRpcServerException, InterruptedException { - // If rpc scheduler indicates back off based on performance - // degradation such as response time or rpc queue is full, - // we will ask the client to back off. - if (callQueue.shouldBackOff(call) || !callQueue.offer(call)) { - rpcMetrics.incrClientBackoff(); - RetriableException retriableException = - new RetriableException("Server is too busy."); - throw new WrappedRpcServerExceptionSuppressed( - RpcErrorCodeProto.ERROR_RPC_SERVER, retriableException); - } - } - /** * Establish RPC connection setup by negotiating SASL if required, then * reading and authorizing the connection header @@ -2487,6 +2489,21 @@ public abstract class Server { } } + public void queueCall(Call call) throws IOException, InterruptedException { + if (!callQueue.isClientBackoffEnabled()) { + callQueue.put(call); // queue the call; maybe blocked here + } else if (callQueue.shouldBackOff(call) || !callQueue.offer(call)) { + // If rpc scheduler indicates back off based on performance degradation + // such as response time or rpc queue is full, we will ask the client + // to back off by throwing RetriableException. Whether the client will + // honor RetriableException and retry depends the client and its policy. + // For example, IPC clients using FailoverOnNetworkExceptionRetry handle + // RetriableException. + rpcMetrics.incrClientBackoff(); + throw new RetriableException("Server is too busy."); + } + } + /** Handles queued calls . */ private class Handler extends Thread { public Handler(int instanceNumber) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java index a9dbb41fd98..3d6d461b855 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java @@ -46,6 +46,7 @@ import org.apache.htrace.core.Tracer; /** An RpcEngine implementation for Writable data. */ @InterfaceStability.Evolving +@Deprecated public class WritableRpcEngine implements RpcEngine { private static final Log LOG = LogFactory.getLog(RPC.class); @@ -331,6 +332,7 @@ public class WritableRpcEngine implements RpcEngine { /** An RPC Server. */ + @Deprecated public static class Server extends RPC.Server { /** * Construct an RPC server. @@ -443,7 +445,8 @@ public class WritableRpcEngine implements RpcEngine { value = value.substring(0, 55)+"..."; LOG.info(value); } - + + @Deprecated static class WritableRpcInvoker implements RpcInvoker { @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java index 4fa839f5e54..79eae123144 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java @@ -47,15 +47,17 @@ import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; import org.apache.hadoop.security.ssl.SSLFactory; +import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.ServletUtil; import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; /** * Change log level in runtime. */ @InterfaceStability.Evolving public class LogLevel { - public static final String USAGES = "\nUsage: General options are:\n" + public static final String USAGES = "\nUsage: Command options are:\n" + "\t[-getlevel [-protocol (http|https)]\n" + "\t[-setlevel " + "[-protocol (http|https)]\n"; @@ -67,7 +69,7 @@ public class LogLevel { */ public static void main(String[] args) throws Exception { CLI cli = new CLI(new Configuration()); - System.exit(cli.run(args)); + System.exit(ToolRunner.run(cli, args)); } /** @@ -81,6 +83,7 @@ public class LogLevel { private static void printUsage() { System.err.println(USAGES); + GenericOptionsParser.printGenericCommandUsage(System.err); } public static boolean isValidProtocol(String protocol) { @@ -107,7 +110,7 @@ public class LogLevel { sendLogLevelRequest(); } catch (HadoopIllegalArgumentException e) { printUsage(); - throw e; + return -1; } return 0; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java index cf5b17678b0..0e6c253c5bc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java @@ -813,7 +813,7 @@ public class NetworkTopology { } } if (numOfDatanodes == 0) { - LOG.warn("Failed to find datanode (scope=\"{}\" excludedScope=\"{}\").", + LOG.debug("Failed to find datanode (scope=\"{}\" excludedScope=\"{}\").", String.valueOf(scope), String.valueOf(excludedScope)); return null; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java index 5a8e81f0229..8e12ef1c538 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java @@ -104,12 +104,8 @@ public class Credentials implements Writable { for (Map.Entry> e : tokenMap.entrySet()) { Token token = e.getValue(); - if (token instanceof Token.PrivateToken && - ((Token.PrivateToken) token).getPublicService().equals(alias)) { - Token privateToken = - new Token.PrivateToken<>(t); - privateToken.setService(token.getService()); - tokensToAdd.put(e.getKey(), privateToken); + if (token.isPrivateCloneOf(alias)) { + tokensToAdd.put(e.getKey(), t.privateClone(token.getService())); } } tokenMap.putAll(tokensToAdd); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KerberosAuthException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KerberosAuthException.java new file mode 100644 index 00000000000..811c7c969b6 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KerberosAuthException.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +import static org.apache.hadoop.security.UGIExceptionMessages.*; + +import java.io.IOException; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Thrown when {@link UserGroupInformation} failed with an unrecoverable error, + * such as failure in kerberos login/logout, invalid subject etc. + * + * Caller should not retry when catching this exception. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class KerberosAuthException extends IOException { + static final long serialVersionUID = 31L; + + private String user; + private String principal; + private String keytabFile; + private String ticketCacheFile; + private String initialMessage; + + public KerberosAuthException(String msg) { + super(msg); + } + + public KerberosAuthException(Throwable cause) { + super(cause); + } + + public KerberosAuthException(String initialMsg, Throwable cause) { + this(cause); + initialMessage = initialMsg; + } + + public void setUser(final String u) { + user = u; + } + + public void setPrincipal(final String p) { + principal = p; + } + + public void setKeytabFile(final String k) { + keytabFile = k; + } + + public void setTicketCacheFile(final String t) { + ticketCacheFile = t; + } + + /** @return The initial message, or null if not set. */ + public String getInitialMessage() { + return initialMessage; + } + + /** @return The keytab file path, or null if not set. */ + public String getKeytabFile() { + return keytabFile; + } + + /** @return The principal, or null if not set. */ + public String getPrincipal() { + return principal; + } + + /** @return The ticket cache file path, or null if not set. */ + public String getTicketCacheFile() { + return ticketCacheFile; + } + + /** @return The user, or null if not set. */ + public String getUser() { + return user; + } + + @Override + public String getMessage() { + final StringBuilder sb = new StringBuilder(); + if (initialMessage != null) { + sb.append(initialMessage); + } + if (user != null) { + sb.append(FOR_USER + user); + } + if (principal != null) { + sb.append(FOR_PRINCIPAL + principal); + } + if (keytabFile != null) { + sb.append(FROM_KEYTAB + keytabFile); + } + if (ticketCacheFile != null) { + sb.append(USING_TICKET_CACHE_FILE+ ticketCacheFile); + } + sb.append(" " + super.getMessage()); + return sb.toString(); + } +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UGIExceptionMessages.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UGIExceptionMessages.java new file mode 100644 index 00000000000..c4d30e509e3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UGIExceptionMessages.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +/** + * Standard strings to use in exception messages + * in {@link KerberosAuthException} when throwing. + */ +final class UGIExceptionMessages { + + public static final String FAILURE_TO_LOGIN = "failure to login:"; + public static final String FOR_USER = " for user: "; + public static final String FOR_PRINCIPAL = " for principal: "; + public static final String FROM_KEYTAB = " from keytab "; + public static final String LOGIN_FAILURE = "Login failure"; + public static final String LOGOUT_FAILURE = "Logout failure"; + public static final String MUST_FIRST_LOGIN = + "login must be done first"; + public static final String MUST_FIRST_LOGIN_FROM_KEYTAB = + "loginUserFromKeyTab must be done first"; + public static final String SUBJECT_MUST_CONTAIN_PRINCIPAL = + "Provided Subject must contain a KerberosPrincipal"; + public static final String SUBJECT_MUST_NOT_BE_NULL = + "Subject must not be null"; + public static final String USING_TICKET_CACHE_FILE = + " using ticket cache file: "; + + //checkstyle: Utility classes should not have a public or default constructor. + private UGIExceptionMessages() { + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index ed3a9d053ea..e8711b03944 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_MET import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES; +import static org.apache.hadoop.security.UGIExceptionMessages.*; import static org.apache.hadoop.util.PlatformName.IBM_JAVA; import java.io.File; @@ -38,7 +39,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -652,33 +652,7 @@ public class UserGroupInformation { } this.isKrbTkt = KerberosUtil.hasKerberosTicket(subject); } - - /** - * Copies the Subject of this UGI and creates a new UGI with the new subject. - * This can be used to add credentials (e.g. tokens) to different copies of - * the same UGI, allowing multiple users with different tokens to reuse the - * UGI without re-authenticating with Kerberos. - * @return clone of the UGI with a new subject. - */ - @InterfaceAudience.Public - @InterfaceStability.Evolving - public UserGroupInformation copySubjectAndUgi() { - Subject subj = getSubject(); - // The ctor will set other fields automatically from the principals. - return new UserGroupInformation(new Subject(false, subj.getPrincipals(), - cloneCredentials(subj.getPublicCredentials()), - cloneCredentials(subj.getPrivateCredentials()))); - } - - private static Set cloneCredentials(Set old) { - Set set = new HashSet<>(); - // Make sure Hadoop credentials objects do not reuse the maps. - for (Object o : old) { - set.add(o instanceof Credentials ? new Credentials((Credentials)o) : o); - } - return set; - } - + /** * checks if logged in using kerberos * @return true if the subject logged via keytab or has a Kerberos TGT @@ -782,8 +756,11 @@ public class UserGroupInformation { ugi.setAuthenticationMethod(AuthenticationMethod.KERBEROS); return ugi; } catch (LoginException le) { - throw new IOException("failure to login using ticket cache file " + - ticketCache, le); + KerberosAuthException kae = + new KerberosAuthException(FAILURE_TO_LOGIN, le); + kae.setUser(user); + kae.setTicketCacheFile(ticketCache); + throw kae; } } @@ -792,16 +769,17 @@ public class UserGroupInformation { * * @param subject The KerberosPrincipal to use in UGI * - * @throws IOException if the kerberos login fails + * @throws IOException + * @throws KerberosAuthException if the kerberos login fails */ public static UserGroupInformation getUGIFromSubject(Subject subject) throws IOException { if (subject == null) { - throw new IOException("Subject must not be null"); + throw new KerberosAuthException(SUBJECT_MUST_NOT_BE_NULL); } if (subject.getPrincipals(KerberosPrincipal.class).isEmpty()) { - throw new IOException("Provided Subject must contain a KerberosPrincipal"); + throw new KerberosAuthException(SUBJECT_MUST_CONTAIN_PRINCIPAL); } KerberosPrincipal principal = @@ -921,7 +899,7 @@ public class UserGroupInformation { loginUser.spawnAutoRenewalThreadForUserCreds(); } catch (LoginException le) { LOG.debug("failure to login", le); - throw new IOException("failure to login: " + le, le); + throw new KerberosAuthException(FAILURE_TO_LOGIN, le); } if (LOG.isDebugEnabled()) { LOG.debug("UGI loginUser:"+loginUser); @@ -968,67 +946,68 @@ public class UserGroupInformation { /**Spawn a thread to do periodic renewals of kerberos credentials*/ private void spawnAutoRenewalThreadForUserCreds() { - if (isSecurityEnabled()) { - //spawn thread only if we have kerb credentials - if (user.getAuthenticationMethod() == AuthenticationMethod.KERBEROS && - !isKeytab) { - Thread t = new Thread(new Runnable() { - - @Override - public void run() { - String cmd = conf.get("hadoop.kerberos.kinit.command", - "kinit"); - KerberosTicket tgt = getTGT(); + if (!isSecurityEnabled() + || user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS + || isKeytab) { + return; + } + + //spawn thread only if we have kerb credentials + Thread t = new Thread(new Runnable() { + + @Override + public void run() { + String cmd = conf.get("hadoop.kerberos.kinit.command", "kinit"); + KerberosTicket tgt = getTGT(); + if (tgt == null) { + return; + } + long nextRefresh = getRefreshTime(tgt); + while (true) { + try { + long now = Time.now(); + if (LOG.isDebugEnabled()) { + LOG.debug("Current time is " + now); + LOG.debug("Next refresh is " + nextRefresh); + } + if (now < nextRefresh) { + Thread.sleep(nextRefresh - now); + } + Shell.execCommand(cmd, "-R"); + if (LOG.isDebugEnabled()) { + LOG.debug("renewed ticket"); + } + reloginFromTicketCache(); + tgt = getTGT(); if (tgt == null) { + LOG.warn("No TGT after renewal. Aborting renew thread for " + + getUserName()); return; } - long nextRefresh = getRefreshTime(tgt); - while (true) { - try { - long now = Time.now(); - if(LOG.isDebugEnabled()) { - LOG.debug("Current time is " + now); - LOG.debug("Next refresh is " + nextRefresh); - } - if (now < nextRefresh) { - Thread.sleep(nextRefresh - now); - } - Shell.execCommand(cmd, "-R"); - if(LOG.isDebugEnabled()) { - LOG.debug("renewed ticket"); - } - reloginFromTicketCache(); - tgt = getTGT(); - if (tgt == null) { - LOG.warn("No TGT after renewal. Aborting renew thread for " + - getUserName()); - return; - } - nextRefresh = Math.max(getRefreshTime(tgt), - now + kerberosMinSecondsBeforeRelogin); - } catch (InterruptedException ie) { - LOG.warn("Terminating renewal thread"); - return; - } catch (IOException ie) { - LOG.warn("Exception encountered while running the" + - " renewal command. Aborting renew thread. " + ie); - return; - } - } + nextRefresh = Math.max(getRefreshTime(tgt), + now + kerberosMinSecondsBeforeRelogin); + } catch (InterruptedException ie) { + LOG.warn("Terminating renewal thread"); + return; + } catch (IOException ie) { + LOG.warn("Exception encountered while running the" + + " renewal command. Aborting renew thread. " + ie); + return; } - }); - t.setDaemon(true); - t.setName("TGT Renewer for " + getUserName()); - t.start(); + } } - } + }); + t.setDaemon(true); + t.setName("TGT Renewer for " + getUserName()); + t.start(); } /** * Log a user in from a keytab file. Loads a user identity from a keytab * file and logs them in. They become the currently logged-in user. * @param user the principal name to load from the keytab * @param path the path to the keytab file - * @throws IOException if the keytab file can't be read + * @throws IOException + * @throws KerberosAuthException if it's a kerberos login exception. */ @InterfaceAudience.Public @InterfaceStability.Evolving @@ -1057,8 +1036,10 @@ public class UserGroupInformation { if (start > 0) { metrics.loginFailure.add(Time.now() - start); } - throw new IOException("Login failure for " + user + " from keytab " + - path+ ": " + le, le); + KerberosAuthException kae = new KerberosAuthException(LOGIN_FAILURE, le); + kae.setUser(user); + kae.setKeytabFile(path); + throw kae; } LOG.info("Login successful for user " + keytabPrincipal + " using keytab file " + keytabFile); @@ -1069,8 +1050,9 @@ public class UserGroupInformation { * This method assumes that the user logged in by calling * {@link #loginUserFromKeytab(String, String)}. * - * @throws IOException if a failure occurred in logout, or if the user did - * not log in by invoking loginUserFromKeyTab() before. + * @throws IOException + * @throws KerberosAuthException if a failure occurred in logout, + * or if the user did not log in by invoking loginUserFromKeyTab() before. */ @InterfaceAudience.Public @InterfaceStability.Evolving @@ -1081,7 +1063,7 @@ public class UserGroupInformation { } LoginContext login = getLogin(); if (login == null || keytabFile == null) { - throw new IOException("loginUserFromKeytab must be done first"); + throw new KerberosAuthException(MUST_FIRST_LOGIN_FROM_KEYTAB); } try { @@ -1092,9 +1074,10 @@ public class UserGroupInformation { login.logout(); } } catch (LoginException le) { - throw new IOException("Logout failure for " + user + " from keytab " + - keytabFile + ": " + le, - le); + KerberosAuthException kae = new KerberosAuthException(LOGOUT_FAILURE, le); + kae.setUser(user.toString()); + kae.setKeytabFile(keytabFile); + throw kae; } LOG.info("Logout successful for user " + keytabPrincipal @@ -1105,6 +1088,7 @@ public class UserGroupInformation { * Re-login a user from keytab if TGT is expired or is close to expiry. * * @throws IOException + * @throws KerberosAuthException if it's a kerberos login exception. */ public synchronized void checkTGTAndReloginFromKeytab() throws IOException { if (!isSecurityEnabled() @@ -1126,12 +1110,12 @@ public class UserGroupInformation { * happened already. * The Subject field of this UserGroupInformation object is updated to have * the new credentials. - * @throws IOException on a failure + * @throws IOException + * @throws KerberosAuthException on a failure */ @InterfaceAudience.Public @InterfaceStability.Evolving - public synchronized void reloginFromKeytab() - throws IOException { + public synchronized void reloginFromKeytab() throws IOException { if (!isSecurityEnabled() || user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS || !isKeytab) @@ -1151,7 +1135,7 @@ public class UserGroupInformation { LoginContext login = getLogin(); if (login == null || keytabFile == null) { - throw new IOException("loginUserFromKeyTab must be done first"); + throw new KerberosAuthException(MUST_FIRST_LOGIN_FROM_KEYTAB); } long start = 0; @@ -1183,8 +1167,10 @@ public class UserGroupInformation { if (start > 0) { metrics.loginFailure.add(Time.now() - start); } - throw new IOException("Login failure for " + keytabPrincipal + - " from keytab " + keytabFile + ": " + le, le); + KerberosAuthException kae = new KerberosAuthException(LOGIN_FAILURE, le); + kae.setPrincipal(keytabPrincipal); + kae.setKeytabFile(keytabFile); + throw kae; } } @@ -1193,19 +1179,19 @@ public class UserGroupInformation { * method assumes that login had happened already. * The Subject field of this UserGroupInformation object is updated to have * the new credentials. - * @throws IOException on a failure + * @throws IOException + * @throws KerberosAuthException on a failure */ @InterfaceAudience.Public @InterfaceStability.Evolving - public synchronized void reloginFromTicketCache() - throws IOException { + public synchronized void reloginFromTicketCache() throws IOException { if (!isSecurityEnabled() || user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS || !isKrbTkt) return; LoginContext login = getLogin(); if (login == null) { - throw new IOException("login must be done first"); + throw new KerberosAuthException(MUST_FIRST_LOGIN); } long now = Time.now(); if (!hasSufficientTimeElapsed(now)) { @@ -1232,8 +1218,9 @@ public class UserGroupInformation { login.login(); setLogin(login); } catch (LoginException le) { - throw new IOException("Login failure for " + getUserName() + ": " + le, - le); + KerberosAuthException kae = new KerberosAuthException(LOGIN_FAILURE, le); + kae.setUser(getUserName()); + throw kae; } } @@ -1279,8 +1266,10 @@ public class UserGroupInformation { if (start > 0) { metrics.loginFailure.add(Time.now() - start); } - throw new IOException("Login failure for " + user + " from keytab " + - path + ": " + le, le); + KerberosAuthException kae = new KerberosAuthException(LOGIN_FAILURE, le); + kae.setUser(user); + kae.setKeytabFile(path); + throw kae; } finally { if(oldKeytabFile != null) keytabFile = oldKeytabFile; if(oldKeytabPrincipal != null) keytabPrincipal = oldKeytabPrincipal; @@ -1611,7 +1600,7 @@ public class UserGroupInformation { Credentials creds = new Credentials(getCredentialsInternal()); Iterator> iter = creds.getAllTokens().iterator(); while (iter.hasNext()) { - if (iter.next() instanceof Token.PrivateToken) { + if (iter.next().isPrivate()) { iter.remove(); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java index 784e7975045..713fb20a3c8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java @@ -222,23 +222,67 @@ public class Token implements Writable { service = newService; } + /** + * Whether this is a private token. + * @return false always for non-private tokens + */ + public boolean isPrivate() { + return false; + } + + /** + * Whether this is a private clone of a public token. + * @param thePublicService the public service name + * @return false always for non-private tokens + */ + public boolean isPrivateCloneOf(Text thePublicService) { + return false; + } + + /** + * Create a private clone of a public token. + * @param newService the new service name + * @return a private token + */ + public Token privateClone(Text newService) { + return new PrivateToken<>(this, newService); + } + /** * Indicates whether the token is a clone. Used by HA failover proxy * to indicate a token should not be visible to the user via * UGI.getCredentials() */ - @InterfaceAudience.Private - @InterfaceStability.Unstable - public static class PrivateToken extends Token { + static class PrivateToken extends Token { final private Text publicService; - public PrivateToken(Token token) { - super(token); - publicService = new Text(token.getService()); + PrivateToken(Token publicToken, Text newService) { + super(publicToken.identifier, publicToken.password, publicToken.kind, + newService); + assert !publicToken.isPrivate(); + publicService = publicToken.service; + if (LOG.isDebugEnabled()) { + LOG.debug("Cloned private token " + this + " from " + publicToken); + } } - public Text getPublicService() { - return publicService; + /** + * Whether this is a private token. + * @return true always for private tokens + */ + @Override + public boolean isPrivate() { + return true; + } + + /** + * Whether this is a private clone of a public token. + * @param thePublicService the public service name + * @return true when the public service is the same as specified + */ + @Override + public boolean isPrivateCloneOf(Text thePublicService) { + return publicService.equals(thePublicService); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 5b8d49d3139..48827282d99 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -93,7 +93,7 @@ The name of the Network Interface from which the service should determine its host name for Kerberos login. e.g. eth2. In a multi-homed environment, - the setting can be used to affect the _HOST subsitution in the service + the setting can be used to affect the _HOST substitution in the service Kerberos principal. If this configuration value is not set, the service will use its default hostname as returned by InetAddress.getLocalHost().getCanonicalHostName(). @@ -400,7 +400,7 @@ The number of levels to go up the group hierarchy when determining which groups a user is part of. 0 Will represent checking just the group that the user belongs to. Each additional level will raise the - time it takes to exectue a query by at most + time it takes to execute a query by at most hadoop.security.group.mapping.ldap.directory.search.timeout. The default will usually be appropriate for all LDAP systems. @@ -1939,7 +1939,7 @@ dr.who=; Static mapping of user to groups. This will override the groups if - available in the system for the specified user. In otherwords, groups + available in the system for the specified user. In other words, groups look-up will not happen for these users, instead groups mapped in this configuration will be used. Mapping should be in this format. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md index 4d7d5044ad5..2ece71a3bea 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md @@ -202,7 +202,9 @@ Manage keys via the KeyProvider. For details on KeyProviders, see the [Transpare Providers frequently require that a password or other secret is supplied. If the provider requires a password and is unable to find one, it will use a default password and emit a warning message that the default password is being used. If the `-strict` flag is supplied, the warning message becomes an error message and the command returns immediately with an error status. -NOTE: Some KeyProviders (e.g. org.apache.hadoop.crypto.key.JavaKeyStoreProvider) does not support uppercase key names. +NOTE: Some KeyProviders (e.g. org.apache.hadoop.crypto.key.JavaKeyStoreProvider) do not support uppercase key names. + +NOTE: Some KeyProviders do not directly execute a key deletion (e.g. performs a soft-delete instead, or delay the actual deletion, to prevent mistake). In these cases, one may encounter errors when creating/deleting a key with the same name after deleting it. Please check the underlying KeyProvider for details. ### `trace` diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index a8a192012c7..ee7bc285694 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -504,7 +504,7 @@ See [HDFS Snapshots Guide](../hadoop-hdfs/HdfsSnapshots.html). rm ---- -Usage: `hadoop fs -rm [-f] [-r |-R] [-skipTrash] URI [URI ...]` +Usage: `hadoop fs -rm [-f] [-r |-R] [-skipTrash] [-safely] URI [URI ...]` Delete files specified as args. @@ -523,6 +523,7 @@ Options: * The -R option deletes the directory and any content under it recursively. * The -r option is equivalent to -R. * The -skipTrash option will bypass trash, if enabled, and delete the specified file(s) immediately. This can be useful when it is necessary to delete files from an over-quota directory. +* The -safely option will require safety confirmation before deleting directory with total number of files greater than `hadoop.shell.delete.limit.num.files` (in core-site.xml, default: 100). It can be used with -skipTrash to prevent accidental deletion of large directories. Delay is expected when walking over large directory recursively to count the number of files to be deleted before the confirmation. Example: diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index 1587842029f..2c9dd5d29da 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -669,19 +669,40 @@ exists in the metadata, but no copies of any its blocks can be located; ### `boolean delete(Path p, boolean recursive)` +Delete a path, be it a file, symbolic link or directory. The +`recursive` flag indicates whether a recursive delete should take place —if +unset then a non-empty directory cannot be deleted. + +Except in the special case of the root directory, if this API call +completed successfully then there is nothing at the end of the path. +That is: the outcome is desired. The return flag simply tells the caller +whether or not any change was made to the state of the filesystem. + +*Note*: many uses of this method surround it with checks for the return value being +false, raising exception if so. For example + +```java +if (!fs.delete(path, true)) throw new IOException("Could not delete " + path); +``` + +This pattern is not needed. Code SHOULD just call `delete(path, recursive)` and +assume the destination is no longer present —except in the special case of root +directories, which will always remain (see below for special coverage of root directories). + #### Preconditions -A directory with children and recursive == false cannot be deleted +A directory with children and `recursive == False` cannot be deleted if isDir(FS, p) and not recursive and (children(FS, p) != {}) : raise IOException +(HDFS raises `PathIsNotEmptyDirectoryException` here.) #### Postconditions ##### Nonexistent path -If the file does not exist the FS state does not change +If the file does not exist the filesystem state does not change if not exists(FS, p): FS' = FS @@ -700,7 +721,7 @@ A path referring to a file is removed, return value: `True` result = True -##### Empty root directory +##### Empty root directory, `recursive == False` Deleting an empty root does not change the filesystem state and may return true or false. @@ -711,7 +732,10 @@ and may return true or false. There is no consistent return code from an attempt to delete the root directory. -##### Empty (non-root) directory +Implementations SHOULD return true; this avoids code which checks for a false +return value from overreacting. + +##### Empty (non-root) directory `recursive == False` Deleting an empty directory that is not root will remove the path from the FS and return true. @@ -721,26 +745,41 @@ return true. result = True -##### Recursive delete of root directory +##### Recursive delete of non-empty root directory Deleting a root path with children and `recursive==True` can do one of two things. -The POSIX model assumes that if the user has +1. The POSIX model assumes that if the user has the correct permissions to delete everything, they are free to do so (resulting in an empty filesystem). - if isDir(FS, p) and isRoot(p) and recursive : - FS' = ({["/"]}, {}, {}, {}) - result = True + if isDir(FS, p) and isRoot(p) and recursive : + FS' = ({["/"]}, {}, {}, {}) + result = True -In contrast, HDFS never permits the deletion of the root of a filesystem; the -filesystem can be taken offline and reformatted if an empty +1. HDFS never permits the deletion of the root of a filesystem; the +filesystem must be taken offline and reformatted if an empty filesystem is desired. - if isDir(FS, p) and isRoot(p) and recursive : - FS' = FS - result = False + if isDir(FS, p) and isRoot(p) and recursive : + FS' = FS + result = False + +HDFS has the notion of *Protected Directories*, which are declared in +the option `fs.protected.directories`. Any attempt to delete such a directory +or a parent thereof raises an `AccessControlException`. Accordingly, any +attempt to delete the root directory SHALL, if there is a protected directory, +result in such an exception being raised. + +This specification does not recommend any specific action. Do note, however, +that the POSIX model assumes that there is a permissions model such that normal +users do not have the permission to delete that root directory; it is an action +which only system administrators should be able to perform. + +Any filesystem client which interacts with a remote filesystem which lacks +such a security model, MAY reject calls to `delete("/", true)` on the basis +that it makes it too easy to lose data. ##### Recursive delete of non-root directory @@ -766,11 +805,11 @@ removes the path and all descendants #### Implementation Notes -* S3N, Swift, FTP and potentially other non-traditional FileSystems -implement `delete()` as recursive listing and file delete operation. -This can break the expectations of client applications -and means that -they cannot be used as drop-in replacements for HDFS. - +* Object Stores and other non-traditional filesystems onto which a directory + tree is emulated, tend to implement `delete()` as recursive listing and +entry-by-entry delete operation. +This can break the expectations of client applications for O(1) atomic directory +deletion, preventing the stores' use as drop-in replacements for HDFS. ### `boolean rename(Path src, Path d)` diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java index ad3d5c5784f..60035be94dc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java @@ -18,11 +18,15 @@ package org.apache.hadoop.conf; import java.io.StringWriter; +import java.io.PrintWriter; import java.io.StringReader; import java.util.HashMap; import java.util.Map; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.ServletConfig; +import javax.servlet.ServletContext; import javax.ws.rs.core.HttpHeaders; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -34,17 +38,36 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; -import junit.framework.TestCase; +import com.google.common.base.Strings; + +import org.apache.hadoop.http.HttpServer2; +import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.mock; +import static org.junit.Assert.*; /** * Basic test case that the ConfServlet can write configuration * to its output in XML and JSON format. */ -public class TestConfServlet extends TestCase { +public class TestConfServlet { private static final String TEST_KEY = "testconfservlet.key"; private static final String TEST_VAL = "testval"; + private static final Map TEST_PROPERTIES = + new HashMap(); + private static final Map TEST_FORMATS = + new HashMap(); + + @BeforeClass + public static void initTestProperties() { + TEST_PROPERTIES.put("test.key1", "value1"); + TEST_PROPERTIES.put("test.key2", "value2"); + TEST_PROPERTIES.put("test.key3", "value3"); + TEST_FORMATS.put(ConfServlet.FORMAT_XML, "application/xml"); + TEST_FORMATS.put(ConfServlet.FORMAT_JSON, "application/json"); + } private Configuration getTestConf() { Configuration testConf = new Configuration(); @@ -52,6 +75,14 @@ public class TestConfServlet extends TestCase { return testConf; } + private Configuration getMultiPropertiesConf() { + Configuration testConf = new Configuration(false); + for(String key : TEST_PROPERTIES.keySet()) { + testConf.set(key, TEST_PROPERTIES.get(key)); + } + return testConf; + } + @Test public void testParseHeaders() throws Exception { HashMap verifyMap = new HashMap(); @@ -71,6 +102,92 @@ public class TestConfServlet extends TestCase { } } + private void verifyGetProperty(Configuration conf, String format, + String propertyName) throws Exception { + StringWriter sw = null; + PrintWriter pw = null; + ConfServlet service = null; + try { + service = new ConfServlet(); + ServletConfig servletConf = mock(ServletConfig.class); + ServletContext context = mock(ServletContext.class); + service.init(servletConf); + when(context.getAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE)) + .thenReturn(conf); + when(service.getServletContext()) + .thenReturn(context); + + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getHeader(HttpHeaders.ACCEPT)) + .thenReturn(TEST_FORMATS.get(format)); + when(request.getParameter("name")) + .thenReturn(propertyName); + + HttpServletResponse response = mock(HttpServletResponse.class); + sw = new StringWriter(); + pw = new PrintWriter(sw); + when(response.getWriter()).thenReturn(pw); + + // response request + service.doGet(request, response); + String result = sw.toString().trim(); + + // if property name is null or empty, expect all properties + // in the response + if (Strings.isNullOrEmpty(propertyName)) { + for(String key : TEST_PROPERTIES.keySet()) { + assertTrue(result.contains(key) && + result.contains(TEST_PROPERTIES.get(key))); + } + } else { + if(conf.get(propertyName) != null) { + // if property name is not empty and property is found + assertTrue(result.contains(propertyName)); + for(String key : TEST_PROPERTIES.keySet()) { + if(!key.equals(propertyName)) { + assertFalse(result.contains(key)); + } + } + } else { + // if property name is not empty, and it's not in configuration + // expect proper error code and error message is set to the response + Mockito.verify(response).sendError( + Mockito.eq(HttpServletResponse.SC_NOT_FOUND), + Mockito.eq("Property " + propertyName + " not found")); + } + } + } finally { + if (sw != null) { + sw.close(); + } + if (pw != null) { + pw.close(); + } + if (service != null) { + service.destroy(); + } + } + } + + @Test + public void testGetProperty() throws Exception { + Configuration configurations = getMultiPropertiesConf(); + // list various of property names + String[] testKeys = new String[] { + "test.key1", + "test.unknown.key", + "", + "test.key2", + null + }; + + for(String format : TEST_FORMATS.keySet()) { + for(String key : testKeys) { + verifyGetProperty(configurations, format, key); + } + } + } + @Test @SuppressWarnings("unchecked") public void testWriteJson() throws Exception { @@ -109,7 +226,6 @@ public class TestConfServlet extends TestCase { for (int i = 0; i < nameNodes.getLength(); i++) { Node nameNode = nameNodes.item(i); String key = nameNode.getTextContent(); - System.err.println("xml key: " + key); if (TEST_KEY.equals(key)) { foundSetting = true; Element propertyElem = (Element)nameNode.getParentNode(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index e7ebea148f4..17112f5c9f6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -42,7 +42,6 @@ import static java.util.concurrent.TimeUnit.*; import junit.framework.TestCase; import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.fail; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration.IntegerRanges; @@ -169,6 +168,9 @@ public class TestConfiguration extends TestCase { declareProperty("my.fullfile", "${my.base}/${my.file}${my.suffix}", "/tmp/hadoop_user/hello.txt"); // check that undefined variables are returned as-is declareProperty("my.failsexpand", "a${my.undefvar}b", "a${my.undefvar}b"); + // check that multiple variable references are resolved + declareProperty("my.user.group", "${user.name} ${user.name}", + "hadoop_user hadoop_user"); endConfig(); Path fileResource = new Path(CONFIG); mock.addResource(fileResource); @@ -1140,7 +1142,19 @@ public class TestConfiguration extends TestCase { this.properties = properties; } } - + + static class SingleJsonConfiguration { + private JsonProperty property; + + public JsonProperty getProperty() { + return property; + } + + public void setProperty(JsonProperty property) { + this.property = property; + } + } + static class JsonProperty { String key; public String getKey() { @@ -1171,7 +1185,14 @@ public class TestConfiguration extends TestCase { boolean isFinal; String resource; } - + + private Configuration getActualConf(String xmlStr) { + Configuration ac = new Configuration(false); + InputStream in = new ByteArrayInputStream(xmlStr.getBytes()); + ac.addResource(in); + return ac; + } + public void testGetSetTrimmedNames() throws IOException { Configuration conf = new Configuration(false); conf.set(" name", "value"); @@ -1180,7 +1201,121 @@ public class TestConfiguration extends TestCase { assertEquals("value", conf.getRaw(" name ")); } - public void testDumpConfiguration () throws IOException { + public void testDumpProperty() throws IOException { + StringWriter outWriter = new StringWriter(); + ObjectMapper mapper = new ObjectMapper(); + String jsonStr = null; + String xmlStr = null; + try { + Configuration testConf = new Configuration(false); + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("test.key1", "value1"); + appendProperty("test.key2", "value2", true); + appendProperty("test.key3", "value3"); + endConfig(); + Path fileResource = new Path(CONFIG); + testConf.addResource(fileResource); + out.close(); + + // case 1: dump an existing property + // test json format + outWriter = new StringWriter(); + Configuration.dumpConfiguration(testConf, "test.key2", outWriter); + jsonStr = outWriter.toString(); + outWriter.close(); + mapper = new ObjectMapper(); + SingleJsonConfiguration jconf1 = + mapper.readValue(jsonStr, SingleJsonConfiguration.class); + JsonProperty jp1 = jconf1.getProperty(); + assertEquals("test.key2", jp1.getKey()); + assertEquals("value2", jp1.getValue()); + assertEquals(true, jp1.isFinal); + assertEquals(fileResource.toUri().getPath(), jp1.getResource()); + + // test xml format + outWriter = new StringWriter(); + testConf.writeXml("test.key2", outWriter); + xmlStr = outWriter.toString(); + outWriter.close(); + Configuration actualConf1 = getActualConf(xmlStr); + assertEquals(1, actualConf1.size()); + assertEquals("value2", actualConf1.get("test.key2")); + assertTrue(actualConf1.getFinalParameters().contains("test.key2")); + assertEquals(fileResource.toUri().getPath(), + actualConf1.getPropertySources("test.key2")[0]); + + // case 2: dump an non existing property + // test json format + try { + outWriter = new StringWriter(); + Configuration.dumpConfiguration(testConf, + "test.unknown.key", outWriter); + outWriter.close(); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + assertTrue(e.getMessage().contains("test.unknown.key") && + e.getMessage().contains("not found")); + } + // test xml format + try { + outWriter = new StringWriter(); + testConf.writeXml("test.unknown.key", outWriter); + outWriter.close(); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + assertTrue(e.getMessage().contains("test.unknown.key") && + e.getMessage().contains("not found")); + } + + // case 3: specify a null property, ensure all configurations are dumped + outWriter = new StringWriter(); + Configuration.dumpConfiguration(testConf, null, outWriter); + jsonStr = outWriter.toString(); + mapper = new ObjectMapper(); + JsonConfiguration jconf3 = + mapper.readValue(jsonStr, JsonConfiguration.class); + assertEquals(3, jconf3.getProperties().length); + + outWriter = new StringWriter(); + testConf.writeXml(null, outWriter); + xmlStr = outWriter.toString(); + outWriter.close(); + Configuration actualConf3 = getActualConf(xmlStr); + assertEquals(3, actualConf3.size()); + assertTrue(actualConf3.getProps().containsKey("test.key1") && + actualConf3.getProps().containsKey("test.key2") && + actualConf3.getProps().containsKey("test.key3")); + + // case 4: specify an empty property, ensure all configurations are dumped + outWriter = new StringWriter(); + Configuration.dumpConfiguration(testConf, "", outWriter); + jsonStr = outWriter.toString(); + mapper = new ObjectMapper(); + JsonConfiguration jconf4 = + mapper.readValue(jsonStr, JsonConfiguration.class); + assertEquals(3, jconf4.getProperties().length); + + outWriter = new StringWriter(); + testConf.writeXml("", outWriter); + xmlStr = outWriter.toString(); + outWriter.close(); + Configuration actualConf4 = getActualConf(xmlStr); + assertEquals(3, actualConf4.size()); + assertTrue(actualConf4.getProps().containsKey("test.key1") && + actualConf4.getProps().containsKey("test.key2") && + actualConf4.getProps().containsKey("test.key3")); + } finally { + if(outWriter != null) { + outWriter.close(); + } + if(out != null) { + out.close(); + } + } + } + + public void testDumpConfiguration() throws IOException { StringWriter outWriter = new StringWriter(); Configuration.dumpConfiguration(conf, outWriter); String jsonStr = outWriter.toString(); @@ -1376,7 +1511,7 @@ public class TestConfiguration extends TestCase { } } - public void testInvalidSubstitutation() { + public void testInvalidSubstitution() { final Configuration configuration = new Configuration(false); // 2-var loops @@ -1390,25 +1525,6 @@ public class TestConfiguration extends TestCase { configuration.set(key, keyExpression); assertEquals("Unexpected value", keyExpression, configuration.get(key)); } - - // - // 3-variable loops - // - - final String expVal1 = "${test.var2}"; - String testVar1 = "test.var1"; - configuration.set(testVar1, expVal1); - configuration.set("test.var2", "${test.var3}"); - configuration.set("test.var3", "${test.var1}"); - assertEquals("Unexpected value", expVal1, configuration.get(testVar1)); - - // 3-variable loop with non-empty value prefix/suffix - // - final String expVal2 = "foo2${test.var2}bar2"; - configuration.set(testVar1, expVal2); - configuration.set("test.var2", "foo3${test.var3}bar3"); - configuration.set("test.var3", "foo1${test.var1}bar1"); - assertEquals("Unexpected value", expVal2, configuration.get(testVar1)); } public void testIncompleteSubbing() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java index 0a6ba651213..a99f7625e0c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java @@ -77,7 +77,9 @@ public abstract class FileContextURIBase { public void tearDown() throws Exception { // Clean up after test completion // No need to clean fc1 as fc1 and fc2 points same location - fc2.delete(BASE, true); + if (fc2 != null) { + fc2.delete(BASE, true); + } } @Test diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java new file mode 100644 index 00000000000..3def5d53880 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; + + +import static org.junit.Assert.assertTrue; + +/** + * Test to make sure df can run and work. + */ +public class TestDFCachingGetSpaceUsed { + final static private File DF_DIR = GenericTestUtils.getTestDir("testdfspace"); + public static final int FILE_SIZE = 1024; + + @Before + public void setUp() { + FileUtil.fullyDelete(DF_DIR); + assertTrue(DF_DIR.mkdirs()); + } + + @After + public void tearDown() throws IOException { + FileUtil.fullyDelete(DF_DIR); + } + + @Test + public void testCanBuildRun() throws Exception { + File file = writeFile("testCanBuild"); + + GetSpaceUsed instance = new CachingGetSpaceUsed.Builder() + .setPath(file) + .setInterval(50060) + .setKlass(DFCachingGetSpaceUsed.class) + .build(); + assertTrue(instance instanceof DFCachingGetSpaceUsed); + assertTrue(instance.getUsed() >= FILE_SIZE - 20); + ((DFCachingGetSpaceUsed) instance).close(); + } + + private File writeFile(String fileName) throws IOException { + File f = new File(DF_DIR, fileName); + assertTrue(f.createNewFile()); + RandomAccessFile randomAccessFile = new RandomAccessFile(f, "rws"); + randomAccessFile.writeUTF(RandomStringUtils.randomAlphabetic(FILE_SIZE)); + randomAccessFile.getFD().sync(); + randomAccessFile.close(); + return f; + } + +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java index 18e8b017c6c..4d627a5e8e2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java @@ -47,16 +47,12 @@ public class TestFileSystemInitialization { @Test public void testMissingLibraries() { - boolean catched = false; try { Configuration conf = new Configuration(); - FileSystem.getFileSystemClass("s3a", conf); - } catch (Exception e) { - catched = true; - } catch (ServiceConfigurationError e) { - // S3A shouldn't find AWS SDK and fail - catched = true; + Class fs = FileSystem.getFileSystemClass("s3a", + conf); + fail("Expected an exception, got a filesystem: " + fs); + } catch (Exception | ServiceConfigurationError expected) { } - assertTrue(catched); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java index 2aba01f8972..338aff6e8d4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java @@ -691,6 +691,10 @@ public class TestTrash extends TestCase { public static class TestTrashPolicy extends TrashPolicy { public TestTrashPolicy() { } + @Override + public void initialize(Configuration conf, FileSystem fs, Path home) { + } + @Override public void initialize(Configuration conf, FileSystem fs) { } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java index cf3ede5c95c..0a8f464486a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java @@ -32,6 +32,8 @@ import org.apache.hadoop.fs.FileStatus; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.deleteChildren; +import static org.apache.hadoop.fs.contract.ContractTestUtils.listChildren; import static org.apache.hadoop.fs.contract.ContractTestUtils.toList; import static org.apache.hadoop.fs.contract.ContractTestUtils.treeWalk; @@ -62,12 +64,40 @@ public abstract class AbstractContractRootDirectoryTest extends AbstractFSContra } @Test - public void testRmEmptyRootDirNonRecursive() throws Throwable { + public void testRmEmptyRootDirRecursive() throws Throwable { //extra sanity checks here to avoid support calls about complete loss of data skipIfUnsupported(TEST_ROOT_TESTS_ENABLED); Path root = new Path("/"); assertIsDirectory(root); boolean deleted = getFileSystem().delete(root, true); + LOG.info("rm -r / of empty dir result is {}", deleted); + assertIsDirectory(root); + } + + @Test + public void testRmEmptyRootDirNonRecursive() throws Throwable { + // extra sanity checks here to avoid support calls about complete loss + // of data + skipIfUnsupported(TEST_ROOT_TESTS_ENABLED); + Path root = new Path("/"); + assertIsDirectory(root); + // make sure it is clean + FileSystem fs = getFileSystem(); + deleteChildren(fs, root, true); + FileStatus[] children = listChildren(fs, root); + if (children.length > 0) { + StringBuilder error = new StringBuilder(); + error.append("Deletion of child entries failed, still have") + .append(children.length) + .append(System.lineSeparator()); + for (FileStatus child : children) { + error.append(" ").append(child.getPath()) + .append(System.lineSeparator()); + } + fail(error.toString()); + } + // then try to delete the empty one + boolean deleted = fs.delete(root, false); LOG.info("rm / of empty dir result is {}", deleted); assertIsDirectory(root); } @@ -88,6 +118,8 @@ public abstract class AbstractContractRootDirectoryTest extends AbstractFSContra } catch (IOException e) { //expected handleExpectedException(e); + // and the file must still be present + assertIsFile(file); } finally { getFileSystem().delete(file, false); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java index 0a1ca496e08..03f47c1b4a7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java @@ -393,6 +393,45 @@ public class ContractTestUtils extends Assert { rejectRootOperation(path, false); } + /** + * List then delete the children of a path, but not the path itself. + * This can be used to delete the entries under a root path when that + * FS does not support {@code delete("/")}. + * @param fileSystem filesystem + * @param path path to delete + * @param recursive flag to indicate child entry deletion should be recursive + * @return the number of child entries found and deleted (not including + * any recursive children of those entries) + * @throws IOException problem in the deletion process. + */ + public static int deleteChildren(FileSystem fileSystem, + Path path, + boolean recursive) throws IOException { + FileStatus[] children = listChildren(fileSystem, path); + for (FileStatus entry : children) { + fileSystem.delete(entry.getPath(), recursive); + } + return children.length; + } + + /** + * List all children of a path, but not the path itself in the case + * that the path refers to a file or empty directory. + * @param fileSystem FS + * @param path path + * @return a list of children, and never the path itself. + * @throws IOException problem in the list process + */ + public static FileStatus[] listChildren(FileSystem fileSystem, + Path path) throws IOException { + FileStatus[] entries = fileSystem.listStatus(path); + if (entries.length == 1 && path.equals(entries[0].getPath())) { + // this is the path: ignore + return new FileStatus[]{}; + } else { + return entries; + } + } public static void noteAction(String action) { if (LOG.isDebugEnabled()) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverController.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverController.java index 164167c43ce..846c8aeb11f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverController.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverController.java @@ -21,6 +21,7 @@ import static org.junit.Assert.*; import java.security.NoSuchAlgorithmException; +import com.google.common.base.Supplier; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; @@ -441,12 +442,16 @@ public class TestZKFailoverController extends ClientBaseWithFixes { cluster.getService(0).getZKFCProxy(conf, 5000).gracefulFailover(); cluster.waitForActiveLockHolder(0); - Thread.sleep(10000); // allow to quiesce + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return cluster.getService(0).fenceCount == 0 && + cluster.getService(1).fenceCount == 0 && + cluster.getService(0).activeTransitionCount == 2 && + cluster.getService(1).activeTransitionCount == 1; + } + }, 100, 60 * 1000); - assertEquals(0, cluster.getService(0).fenceCount); - assertEquals(0, cluster.getService(1).fenceCount); - assertEquals(2, cluster.getService(0).activeTransitionCount); - assertEquals(1, cluster.getService(1).activeTransitionCount); } @Test @@ -590,14 +595,17 @@ public class TestZKFailoverController extends ClientBaseWithFixes { cluster.getService(0).getZKFCProxy(conf, 5000).gracefulFailover(); cluster.waitForActiveLockHolder(0); - Thread.sleep(10000); // allow to quiesce - - assertEquals(0, cluster.getService(0).fenceCount); - assertEquals(0, cluster.getService(1).fenceCount); - assertEquals(0, cluster.getService(2).fenceCount); - assertEquals(2, cluster.getService(0).activeTransitionCount); - assertEquals(1, cluster.getService(1).activeTransitionCount); - assertEquals(1, cluster.getService(2).activeTransitionCount); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return cluster.getService(0).fenceCount == 0 && + cluster.getService(1).fenceCount == 0 && + cluster.getService(2).fenceCount == 0 && + cluster.getService(0).activeTransitionCount == 2 && + cluster.getService(1).activeTransitionCount == 1 && + cluster.getService(2).activeTransitionCount == 1; + } + }, 100, 60 * 1000); } private int runFC(DummyHAService target, String ... args) throws Exception { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index ff6b25e4476..72b603aa29b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -64,6 +64,7 @@ import java.net.ConnectException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -71,6 +72,7 @@ import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -926,6 +928,91 @@ public class TestRPC extends TestRpcBase { } } + @Test(timeout=30000) + public void testExternalCall() throws Exception { + final UserGroupInformation ugi = UserGroupInformation + .createUserForTesting("user123", new String[0]); + final IOException expectedIOE = new IOException("boom"); + + // use 1 handler so the callq can be plugged + final Server server = setupTestServer(conf, 1); + try { + final AtomicBoolean result = new AtomicBoolean(); + + ExternalCall remoteUserCall = newExtCall(ugi, + new PrivilegedExceptionAction() { + @Override + public String run() throws Exception { + return UserGroupInformation.getCurrentUser().getUserName(); + } + }); + + ExternalCall exceptionCall = newExtCall(ugi, + new PrivilegedExceptionAction() { + @Override + public String run() throws Exception { + throw expectedIOE; + } + }); + + final CountDownLatch latch = new CountDownLatch(1); + final CyclicBarrier barrier = new CyclicBarrier(2); + + ExternalCall barrierCall = newExtCall(ugi, + new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + // notify we are in a handler and then wait to keep the callq + // plugged up + latch.countDown(); + barrier.await(); + return null; + } + }); + + server.queueCall(barrierCall); + server.queueCall(exceptionCall); + server.queueCall(remoteUserCall); + + // wait for barrier call to enter the handler, check that the other 2 + // calls are actually queued + latch.await(); + assertEquals(2, server.getCallQueueLen()); + + // unplug the callq + barrier.await(); + barrierCall.get(); + + // verify correct ugi is used + String answer = remoteUserCall.get(); + assertEquals(ugi.getUserName(), answer); + + try { + exceptionCall.get(); + fail("didn't throw"); + } catch (ExecutionException ee) { + assertTrue((ee.getCause()) instanceof IOException); + assertEquals(expectedIOE.getMessage(), ee.getCause().getMessage()); + } + } finally { + server.stop(); + } + } + + private ExternalCall newExtCall(UserGroupInformation ugi, + PrivilegedExceptionAction callable) { + return new ExternalCall(callable) { + @Override + public String getProtocol() { + return "test"; + } + @Override + public UserGroupInformation getRemoteUser() { + return ugi; + } + }; + } + @Test public void testRpcMetrics() throws Exception { Server server; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java index e45d70db746..a52cd46f826 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java @@ -50,7 +50,6 @@ import java.security.PrivilegedExceptionAction; import java.util.Collection; import java.util.ConcurrentModificationException; import java.util.LinkedHashSet; -import java.util.List; import java.util.Set; import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS; @@ -891,40 +890,16 @@ public class TestUserGroupInformation { ugi.addToken(new Text("regular-token"), token); // Now add cloned private token - ugi.addToken(new Text("private-token"), new Token.PrivateToken(token)); - ugi.addToken(new Text("private-token1"), new Token.PrivateToken(token)); + Text service = new Text("private-token"); + ugi.addToken(service, token.privateClone(service)); + Text service1 = new Text("private-token1"); + ugi.addToken(service1, token.privateClone(service1)); // Ensure only non-private tokens are returned Collection> tokens = ugi.getCredentials().getAllTokens(); assertEquals(1, tokens.size()); } - @Test(timeout = 30000) - public void testCopySubjectAndUgi() throws IOException { - SecurityUtil.setAuthenticationMethod(AuthenticationMethod.SIMPLE, conf); - UserGroupInformation.setConfiguration(conf); - UserGroupInformation u1 = UserGroupInformation.getLoginUser(); - assertNotNull(u1); - @SuppressWarnings("unchecked") - Token tmpToken = mock(Token.class); - u1.addToken(tmpToken); - - UserGroupInformation u2 = u1.copySubjectAndUgi(); - assertEquals(u1.getAuthenticationMethod(), u2.getAuthenticationMethod()); - assertNotSame(u1.getSubject(), u2.getSubject()); - Credentials c1 = u1.getCredentials(), c2 = u2.getCredentials(); - List sc1 = c1.getAllSecretKeys(), sc2 = c2.getAllSecretKeys(); - assertArrayEquals(sc1.toArray(new Text[0]), sc2.toArray(new Text[0])); - Collection> ts1 = c1.getAllTokens(), - ts2 = c2.getAllTokens(); - assertArrayEquals(ts1.toArray(new Token[0]), ts2.toArray(new Token[0])); - @SuppressWarnings("unchecked") - Token token = mock(Token.class); - u2.addToken(token); - assertTrue(u2.getCredentials().getAllTokens().contains(token)); - assertFalse(u1.getCredentials().getAllTokens().contains(token)); - } - /** * This test checks a race condition between getting and adding tokens for * the current user. Calling UserGroupInformation.getCurrentUser() returns diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java index f069fcaebda..d8755ec3d97 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java @@ -28,6 +28,8 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.crypto.key.kms.KMSClientProvider; import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.ws.rs.Consumes; @@ -68,6 +70,8 @@ public class KMS { private KeyProviderCryptoExtension provider; private KMSAudit kmsAudit; + private static final Logger LOG = LoggerFactory.getLogger(KMS.class); + public KMS() throws Exception { provider = KMSWebApp.getKeyProvider(); kmsAudit= KMSWebApp.getKMSAudit(); @@ -77,7 +81,7 @@ public class KMS { KMSOp operation) throws AccessControlException { KMSWebApp.getACLs().assertAccess(aclType, ugi, operation, null); } - + private void assertAccess(KMSACLs.Type aclType, UserGroupInformation ugi, KMSOp operation, String key) throws AccessControlException { KMSWebApp.getACLs().assertAccess(aclType, ugi, operation, key); @@ -100,83 +104,101 @@ public class KMS { @Produces(MediaType.APPLICATION_JSON) @SuppressWarnings("unchecked") public Response createKey(Map jsonKey) throws Exception { - KMSWebApp.getAdminCallsMeter().mark(); - UserGroupInformation user = HttpUserGroupInformation.get(); - final String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD); - KMSClientProvider.checkNotEmpty(name, KMSRESTConstants.NAME_FIELD); - assertAccess(KMSACLs.Type.CREATE, user, KMSOp.CREATE_KEY, name); - String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD); - final String material = (String) jsonKey.get(KMSRESTConstants.MATERIAL_FIELD); - int length = (jsonKey.containsKey(KMSRESTConstants.LENGTH_FIELD)) - ? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0; - String description = (String) - jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD); - Map attributes = (Map) - jsonKey.get(KMSRESTConstants.ATTRIBUTES_FIELD); - if (material != null) { - assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user, - KMSOp.CREATE_KEY, name); - } - final KeyProvider.Options options = new KeyProvider.Options( - KMSWebApp.getConfiguration()); - if (cipher != null) { - options.setCipher(cipher); - } - if (length != 0) { - options.setBitLength(length); - } - options.setDescription(description); - options.setAttributes(attributes); + try{ + LOG.trace("Entering createKey Method."); + KMSWebApp.getAdminCallsMeter().mark(); + UserGroupInformation user = HttpUserGroupInformation.get(); + final String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD); + KMSClientProvider.checkNotEmpty(name, KMSRESTConstants.NAME_FIELD); + assertAccess(KMSACLs.Type.CREATE, user, KMSOp.CREATE_KEY, name); + String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD); + final String material; + material = (String) jsonKey.get(KMSRESTConstants.MATERIAL_FIELD); + int length = (jsonKey.containsKey(KMSRESTConstants.LENGTH_FIELD)) + ? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0; + String description = (String) + jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD); + LOG.debug("Creating key with name {}, cipher being used{}, " + + "length of key {}, description of key {}", name, cipher, + length, description); + Map attributes = (Map) + jsonKey.get(KMSRESTConstants.ATTRIBUTES_FIELD); + if (material != null) { + assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user, + KMSOp.CREATE_KEY, name); + } + final KeyProvider.Options options = new KeyProvider.Options( + KMSWebApp.getConfiguration()); + if (cipher != null) { + options.setCipher(cipher); + } + if (length != 0) { + options.setBitLength(length); + } + options.setDescription(description); + options.setAttributes(attributes); - KeyProvider.KeyVersion keyVersion = user.doAs( - new PrivilegedExceptionAction() { - @Override - public KeyVersion run() throws Exception { - KeyProvider.KeyVersion keyVersion = (material != null) - ? provider.createKey(name, Base64.decodeBase64(material), options) - : provider.createKey(name, options); - provider.flush(); - return keyVersion; + KeyProvider.KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyVersion run() throws Exception { + KeyProvider.KeyVersion keyVersion = (material != null) + ? provider.createKey(name, Base64.decodeBase64(material), + options) + : provider.createKey(name, options); + provider.flush(); + return keyVersion; + } } - } - ); + ); - kmsAudit.ok(user, KMSOp.CREATE_KEY, name, "UserProvidedMaterial:" + - (material != null) + " Description:" + description); + kmsAudit.ok(user, KMSOp.CREATE_KEY, name, "UserProvidedMaterial:" + + (material != null) + " Description:" + description); - if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user)) { - keyVersion = removeKeyMaterial(keyVersion); + if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user)) { + keyVersion = removeKeyMaterial(keyVersion); + } + Map json = KMSServerJSONUtils.toJSON(keyVersion); + String requestURL = KMSMDCFilter.getURL(); + int idx = requestURL.lastIndexOf(KMSRESTConstants.KEYS_RESOURCE); + requestURL = requestURL.substring(0, idx); + LOG.trace("Exiting createKey Method."); + return Response.created(getKeyURI(KMSRESTConstants.SERVICE_VERSION, name)) + .type(MediaType.APPLICATION_JSON) + .header("Location", getKeyURI(requestURL, name)).entity(json).build(); + } catch (Exception e) { + LOG.debug("Exception in createKey.", e); + throw e; } - Map json = KMSServerJSONUtils.toJSON(keyVersion); - String requestURL = KMSMDCFilter.getURL(); - int idx = requestURL.lastIndexOf(KMSRESTConstants.KEYS_RESOURCE); - requestURL = requestURL.substring(0, idx); - return Response.created(getKeyURI(KMSRESTConstants.SERVICE_VERSION, name)) - .type(MediaType.APPLICATION_JSON) - .header("Location", getKeyURI(requestURL, name)).entity(json).build(); } @DELETE @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") public Response deleteKey(@PathParam("name") final String name) throws Exception { - KMSWebApp.getAdminCallsMeter().mark(); - UserGroupInformation user = HttpUserGroupInformation.get(); - assertAccess(KMSACLs.Type.DELETE, user, KMSOp.DELETE_KEY, name); - KMSClientProvider.checkNotEmpty(name, "name"); + try { + LOG.trace("Entering deleteKey method."); + KMSWebApp.getAdminCallsMeter().mark(); + UserGroupInformation user = HttpUserGroupInformation.get(); + assertAccess(KMSACLs.Type.DELETE, user, KMSOp.DELETE_KEY, name); + KMSClientProvider.checkNotEmpty(name, "name"); + LOG.debug("Deleting key with name {}.", name); + user.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + provider.deleteKey(name); + provider.flush(); + return null; + } + }); - user.doAs(new PrivilegedExceptionAction() { - @Override - public Void run() throws Exception { - provider.deleteKey(name); - provider.flush(); - return null; - } - }); - - kmsAudit.ok(user, KMSOp.DELETE_KEY, name, ""); - - return Response.ok().build(); + kmsAudit.ok(user, KMSOp.DELETE_KEY, name, ""); + LOG.trace("Exiting deleteKey method."); + return Response.ok().build(); + } catch (Exception e) { + LOG.debug("Exception in deleteKey.", e); + throw e; + } } @POST @@ -185,38 +207,49 @@ public class KMS { @Produces(MediaType.APPLICATION_JSON) public Response rolloverKey(@PathParam("name") final String name, Map jsonMaterial) throws Exception { - KMSWebApp.getAdminCallsMeter().mark(); - UserGroupInformation user = HttpUserGroupInformation.get(); - assertAccess(KMSACLs.Type.ROLLOVER, user, KMSOp.ROLL_NEW_VERSION, name); - KMSClientProvider.checkNotEmpty(name, "name"); - final String material = (String) - jsonMaterial.get(KMSRESTConstants.MATERIAL_FIELD); - if (material != null) { - assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user, - KMSOp.ROLL_NEW_VERSION, name); + try { + LOG.trace("Entering rolloverKey Method."); + KMSWebApp.getAdminCallsMeter().mark(); + UserGroupInformation user = HttpUserGroupInformation.get(); + assertAccess(KMSACLs.Type.ROLLOVER, user, KMSOp.ROLL_NEW_VERSION, name); + KMSClientProvider.checkNotEmpty(name, "name"); + LOG.debug("Rolling key with name {}.", name); + final String material = (String) + jsonMaterial.get(KMSRESTConstants.MATERIAL_FIELD); + if (material != null) { + assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user, + KMSOp.ROLL_NEW_VERSION, name); + } + + KeyProvider.KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyVersion run() throws Exception { + KeyVersion keyVersion = (material != null) + ? provider.rollNewVersion(name, + Base64.decodeBase64(material)) + : provider.rollNewVersion(name); + provider.flush(); + return keyVersion; + } + } + ); + + kmsAudit.ok(user, KMSOp.ROLL_NEW_VERSION, name, "UserProvidedMaterial:" + + (material != null) + + " NewVersion:" + keyVersion.getVersionName()); + + if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user)) { + keyVersion = removeKeyMaterial(keyVersion); + } + Map json = KMSServerJSONUtils.toJSON(keyVersion); + LOG.trace("Exiting rolloverKey Method."); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json) + .build(); + } catch (Exception e) { + LOG.debug("Exception in rolloverKey.", e); + throw e; } - - KeyProvider.KeyVersion keyVersion = user.doAs( - new PrivilegedExceptionAction() { - @Override - public KeyVersion run() throws Exception { - KeyVersion keyVersion = (material != null) - ? provider.rollNewVersion(name, Base64.decodeBase64(material)) - : provider.rollNewVersion(name); - provider.flush(); - return keyVersion; - } - } - ); - - kmsAudit.ok(user, KMSOp.ROLL_NEW_VERSION, name, "UserProvidedMaterial:" + - (material != null) + " NewVersion:" + keyVersion.getVersionName()); - - if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user)) { - keyVersion = removeKeyMaterial(keyVersion); - } - Map json = KMSServerJSONUtils.toJSON(keyVersion); - return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); } @GET @@ -224,52 +257,76 @@ public class KMS { @Produces(MediaType.APPLICATION_JSON) public Response getKeysMetadata(@QueryParam(KMSRESTConstants.KEY) List keyNamesList) throws Exception { - KMSWebApp.getAdminCallsMeter().mark(); - UserGroupInformation user = HttpUserGroupInformation.get(); - final String[] keyNames = keyNamesList.toArray( - new String[keyNamesList.size()]); - assertAccess(KMSACLs.Type.GET_METADATA, user, KMSOp.GET_KEYS_METADATA); + try { + LOG.trace("Entering getKeysMetadata method."); + KMSWebApp.getAdminCallsMeter().mark(); + UserGroupInformation user = HttpUserGroupInformation.get(); + final String[] keyNames = keyNamesList.toArray( + new String[keyNamesList.size()]); + assertAccess(KMSACLs.Type.GET_METADATA, user, KMSOp.GET_KEYS_METADATA); - KeyProvider.Metadata[] keysMeta = user.doAs( - new PrivilegedExceptionAction() { - @Override - public KeyProvider.Metadata[] run() throws Exception { - return provider.getKeysMetadata(keyNames); - } - } - ); + KeyProvider.Metadata[] keysMeta = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyProvider.Metadata[] run() throws Exception { + return provider.getKeysMetadata(keyNames); + } + } + ); - Object json = KMSServerJSONUtils.toJSON(keyNames, keysMeta); - kmsAudit.ok(user, KMSOp.GET_KEYS_METADATA, ""); - return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + Object json = KMSServerJSONUtils.toJSON(keyNames, keysMeta); + kmsAudit.ok(user, KMSOp.GET_KEYS_METADATA, ""); + LOG.trace("Exiting getKeysMetadata method."); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json) + .build(); + } catch (Exception e) { + LOG.debug("Exception in getKeysmetadata.", e); + throw e; + } } @GET @Path(KMSRESTConstants.KEYS_NAMES_RESOURCE) @Produces(MediaType.APPLICATION_JSON) public Response getKeyNames() throws Exception { - KMSWebApp.getAdminCallsMeter().mark(); - UserGroupInformation user = HttpUserGroupInformation.get(); - assertAccess(KMSACLs.Type.GET_KEYS, user, KMSOp.GET_KEYS); + try { + LOG.trace("Entering getKeyNames method."); + KMSWebApp.getAdminCallsMeter().mark(); + UserGroupInformation user = HttpUserGroupInformation.get(); + assertAccess(KMSACLs.Type.GET_KEYS, user, KMSOp.GET_KEYS); - List json = user.doAs( - new PrivilegedExceptionAction>() { - @Override - public List run() throws Exception { - return provider.getKeys(); - } - } - ); + List json = user.doAs( + new PrivilegedExceptionAction>() { + @Override + public List run() throws Exception { + return provider.getKeys(); + } + } + ); - kmsAudit.ok(user, KMSOp.GET_KEYS, ""); - return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + kmsAudit.ok(user, KMSOp.GET_KEYS, ""); + LOG.trace("Exiting getKeyNames method."); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json) + .build(); + } catch (Exception e) { + LOG.debug("Exception in getkeyNames.", e); + throw e; + } } @GET @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") public Response getKey(@PathParam("name") String name) throws Exception { - return getMetadata(name); + try { + LOG.trace("Entering getKey method."); + LOG.debug("Getting key information for key with name {}.", name); + LOG.trace("Exiting getKey method."); + return getMetadata(name); + } catch (Exception e) { + LOG.debug("Exception in getKey.", e); + throw e; + } } @GET @@ -278,23 +335,32 @@ public class KMS { @Produces(MediaType.APPLICATION_JSON) public Response getMetadata(@PathParam("name") final String name) throws Exception { - UserGroupInformation user = HttpUserGroupInformation.get(); - KMSClientProvider.checkNotEmpty(name, "name"); - KMSWebApp.getAdminCallsMeter().mark(); - assertAccess(KMSACLs.Type.GET_METADATA, user, KMSOp.GET_METADATA, name); + try { + LOG.trace("Entering getMetadata method."); + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(name, "name"); + KMSWebApp.getAdminCallsMeter().mark(); + assertAccess(KMSACLs.Type.GET_METADATA, user, KMSOp.GET_METADATA, name); + LOG.debug("Getting metadata for key with name {}.", name); - KeyProvider.Metadata metadata = user.doAs( - new PrivilegedExceptionAction() { - @Override - public KeyProvider.Metadata run() throws Exception { - return provider.getMetadata(name); - } - } - ); + KeyProvider.Metadata metadata = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyProvider.Metadata run() throws Exception { + return provider.getMetadata(name); + } + } + ); - Object json = KMSServerJSONUtils.toJSON(name, metadata); - kmsAudit.ok(user, KMSOp.GET_METADATA, name, ""); - return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + Object json = KMSServerJSONUtils.toJSON(name, metadata); + kmsAudit.ok(user, KMSOp.GET_METADATA, name, ""); + LOG.trace("Exiting getMetadata method."); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json) + .build(); + } catch (Exception e) { + LOG.debug("Exception in getMetadata.", e); + throw e; + } } @GET @@ -303,23 +369,32 @@ public class KMS { @Produces(MediaType.APPLICATION_JSON) public Response getCurrentVersion(@PathParam("name") final String name) throws Exception { - UserGroupInformation user = HttpUserGroupInformation.get(); - KMSClientProvider.checkNotEmpty(name, "name"); - KMSWebApp.getKeyCallsMeter().mark(); - assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_CURRENT_KEY, name); + try { + LOG.trace("Entering getCurrentVersion method."); + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(name, "name"); + KMSWebApp.getKeyCallsMeter().mark(); + assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_CURRENT_KEY, name); + LOG.debug("Getting key version for key with name {}.", name); - KeyVersion keyVersion = user.doAs( - new PrivilegedExceptionAction() { - @Override - public KeyVersion run() throws Exception { - return provider.getCurrentKey(name); - } - } - ); + KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyVersion run() throws Exception { + return provider.getCurrentKey(name); + } + } + ); - Object json = KMSServerJSONUtils.toJSON(keyVersion); - kmsAudit.ok(user, KMSOp.GET_CURRENT_KEY, name, ""); - return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + Object json = KMSServerJSONUtils.toJSON(keyVersion); + kmsAudit.ok(user, KMSOp.GET_CURRENT_KEY, name, ""); + LOG.trace("Exiting getCurrentVersion method."); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json) + .build(); + } catch (Exception e) { + LOG.debug("Exception in getCurrentVersion.", e); + throw e; + } } @GET @@ -327,25 +402,34 @@ public class KMS { @Produces(MediaType.APPLICATION_JSON) public Response getKeyVersion( @PathParam("versionName") final String versionName) throws Exception { - UserGroupInformation user = HttpUserGroupInformation.get(); - KMSClientProvider.checkNotEmpty(versionName, "versionName"); - KMSWebApp.getKeyCallsMeter().mark(); - assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_KEY_VERSION); + try { + LOG.trace("Entering getKeyVersion method."); + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(versionName, "versionName"); + KMSWebApp.getKeyCallsMeter().mark(); + assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_KEY_VERSION); + LOG.debug("Getting key with version name {}.", versionName); - KeyVersion keyVersion = user.doAs( - new PrivilegedExceptionAction() { - @Override - public KeyVersion run() throws Exception { - return provider.getKeyVersion(versionName); - } - } - ); + KeyVersion keyVersion = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyVersion run() throws Exception { + return provider.getKeyVersion(versionName); + } + } + ); - if (keyVersion != null) { - kmsAudit.ok(user, KMSOp.GET_KEY_VERSION, keyVersion.getName(), ""); + if (keyVersion != null) { + kmsAudit.ok(user, KMSOp.GET_KEY_VERSION, keyVersion.getName(), ""); + } + Object json = KMSServerJSONUtils.toJSON(keyVersion); + LOG.trace("Exiting getKeyVersion method."); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json) + .build(); + } catch (Exception e) { + LOG.debug("Exception in getKeyVersion.", e); + throw e; } - Object json = KMSServerJSONUtils.toJSON(keyVersion); - return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); } @SuppressWarnings({ "rawtypes", "unchecked" }) @@ -359,46 +443,65 @@ public class KMS { @DefaultValue("1") @QueryParam(KMSRESTConstants.EEK_NUM_KEYS) final int numKeys) throws Exception { - UserGroupInformation user = HttpUserGroupInformation.get(); - KMSClientProvider.checkNotEmpty(name, "name"); - KMSClientProvider.checkNotNull(edekOp, "eekOp"); + try { + LOG.trace("Entering generateEncryptedKeys method."); + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(name, "name"); + KMSClientProvider.checkNotNull(edekOp, "eekOp"); + LOG.debug("Generating encrypted key with name {}," + + " the edek Operation is {}.", name, edekOp); - Object retJSON; - if (edekOp.equals(KMSRESTConstants.EEK_GENERATE)) { - assertAccess(KMSACLs.Type.GENERATE_EEK, user, KMSOp.GENERATE_EEK, name); + Object retJSON; + if (edekOp.equals(KMSRESTConstants.EEK_GENERATE)) { + LOG.debug("edek Operation is Generate."); + assertAccess(KMSACLs.Type.GENERATE_EEK, user, KMSOp.GENERATE_EEK, name); - final List retEdeks = - new LinkedList(); - try { + final List retEdeks = + new LinkedList(); + try { - user.doAs( - new PrivilegedExceptionAction() { - @Override - public Void run() throws Exception { - for (int i = 0; i < numKeys; i++) { - retEdeks.add(provider.generateEncryptedKey(name)); + user.doAs( + new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + LOG.debug("Generated Encrypted key for {} number of " + + "keys.", numKeys); + for (int i = 0; i < numKeys; i++) { + retEdeks.add(provider.generateEncryptedKey(name)); + } + return null; + } } - return null; - } - } - ); + ); - } catch (Exception e) { - throw new IOException(e); + } catch (Exception e) { + LOG.error("Exception in generateEncryptedKeys:", e); + throw new IOException(e); + } + kmsAudit.ok(user, KMSOp.GENERATE_EEK, name, ""); + retJSON = new ArrayList(); + for (EncryptedKeyVersion edek : retEdeks) { + ((ArrayList) retJSON).add(KMSServerJSONUtils.toJSON(edek)); + } + } else { + StringBuilder error; + error = new StringBuilder("IllegalArgumentException Wrong "); + error.append(KMSRESTConstants.EEK_OP); + error.append(" value, it must be "); + error.append(KMSRESTConstants.EEK_GENERATE); + error.append(" or "); + error.append(KMSRESTConstants.EEK_DECRYPT); + LOG.error(error.toString()); + throw new IllegalArgumentException(error.toString()); } - kmsAudit.ok(user, KMSOp.GENERATE_EEK, name, ""); - retJSON = new ArrayList(); - for (EncryptedKeyVersion edek : retEdeks) { - ((ArrayList)retJSON).add(KMSServerJSONUtils.toJSON(edek)); - } - } else { - throw new IllegalArgumentException("Wrong " + KMSRESTConstants.EEK_OP + - " value, it must be " + KMSRESTConstants.EEK_GENERATE + " or " + - KMSRESTConstants.EEK_DECRYPT); + KMSWebApp.getGenerateEEKCallsMeter().mark(); + LOG.trace("Exiting generateEncryptedKeys method."); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON) + .build(); + } catch (Exception e) { + LOG.debug("Exception in generateEncryptedKeys.", e); + throw e; } - KMSWebApp.getGenerateEEKCallsMeter().mark(); - return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON) - .build(); } @SuppressWarnings("rawtypes") @@ -411,47 +514,64 @@ public class KMS { @QueryParam(KMSRESTConstants.EEK_OP) String eekOp, Map jsonPayload) throws Exception { - UserGroupInformation user = HttpUserGroupInformation.get(); - KMSClientProvider.checkNotEmpty(versionName, "versionName"); - KMSClientProvider.checkNotNull(eekOp, "eekOp"); + try { + LOG.trace("Entering decryptEncryptedKey method."); + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(versionName, "versionName"); + KMSClientProvider.checkNotNull(eekOp, "eekOp"); + LOG.debug("Decrypting key for {}, the edek Operation is {}.", + versionName, eekOp); - final String keyName = (String) jsonPayload.get( - KMSRESTConstants.NAME_FIELD); - String ivStr = (String) jsonPayload.get(KMSRESTConstants.IV_FIELD); - String encMaterialStr = - (String) jsonPayload.get(KMSRESTConstants.MATERIAL_FIELD); - Object retJSON; - if (eekOp.equals(KMSRESTConstants.EEK_DECRYPT)) { - assertAccess(KMSACLs.Type.DECRYPT_EEK, user, KMSOp.DECRYPT_EEK, keyName); - KMSClientProvider.checkNotNull(ivStr, KMSRESTConstants.IV_FIELD); - final byte[] iv = Base64.decodeBase64(ivStr); - KMSClientProvider.checkNotNull(encMaterialStr, - KMSRESTConstants.MATERIAL_FIELD); - final byte[] encMaterial = Base64.decodeBase64(encMaterialStr); + final String keyName = (String) jsonPayload.get( + KMSRESTConstants.NAME_FIELD); + String ivStr = (String) jsonPayload.get(KMSRESTConstants.IV_FIELD); + String encMaterialStr = + (String) jsonPayload.get(KMSRESTConstants.MATERIAL_FIELD); + Object retJSON; + if (eekOp.equals(KMSRESTConstants.EEK_DECRYPT)) { + assertAccess(KMSACLs.Type.DECRYPT_EEK, user, KMSOp.DECRYPT_EEK, + keyName); + KMSClientProvider.checkNotNull(ivStr, KMSRESTConstants.IV_FIELD); + final byte[] iv = Base64.decodeBase64(ivStr); + KMSClientProvider.checkNotNull(encMaterialStr, + KMSRESTConstants.MATERIAL_FIELD); + final byte[] encMaterial = Base64.decodeBase64(encMaterialStr); - KeyProvider.KeyVersion retKeyVersion = user.doAs( - new PrivilegedExceptionAction() { - @Override - public KeyVersion run() throws Exception { - return provider.decryptEncryptedKey( - new KMSClientProvider.KMSEncryptedKeyVersion(keyName, - versionName, iv, KeyProviderCryptoExtension.EEK, - encMaterial) - ); - } - } - ); + KeyProvider.KeyVersion retKeyVersion = user.doAs( + new PrivilegedExceptionAction() { + @Override + public KeyVersion run() throws Exception { + return provider.decryptEncryptedKey( + new KMSClientProvider.KMSEncryptedKeyVersion( + keyName, versionName, iv, + KeyProviderCryptoExtension.EEK, + encMaterial) + ); + } + } + ); - retJSON = KMSServerJSONUtils.toJSON(retKeyVersion); - kmsAudit.ok(user, KMSOp.DECRYPT_EEK, keyName, ""); - } else { - throw new IllegalArgumentException("Wrong " + KMSRESTConstants.EEK_OP + - " value, it must be " + KMSRESTConstants.EEK_GENERATE + " or " + - KMSRESTConstants.EEK_DECRYPT); + retJSON = KMSServerJSONUtils.toJSON(retKeyVersion); + kmsAudit.ok(user, KMSOp.DECRYPT_EEK, keyName, ""); + } else { + StringBuilder error; + error = new StringBuilder("IllegalArgumentException Wrong "); + error.append(KMSRESTConstants.EEK_OP); + error.append(" value, it must be "); + error.append(KMSRESTConstants.EEK_GENERATE); + error.append(" or "); + error.append(KMSRESTConstants.EEK_DECRYPT); + LOG.error(error.toString()); + throw new IllegalArgumentException(error.toString()); + } + KMSWebApp.getDecryptEEKCallsMeter().mark(); + LOG.trace("Exiting decryptEncryptedKey method."); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON) + .build(); + } catch (Exception e) { + LOG.debug("Exception in decryptEncryptedKey.", e); + throw e; } - KMSWebApp.getDecryptEEKCallsMeter().mark(); - return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON) - .build(); } @GET @@ -460,23 +580,32 @@ public class KMS { @Produces(MediaType.APPLICATION_JSON) public Response getKeyVersions(@PathParam("name") final String name) throws Exception { - UserGroupInformation user = HttpUserGroupInformation.get(); - KMSClientProvider.checkNotEmpty(name, "name"); - KMSWebApp.getKeyCallsMeter().mark(); - assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_KEY_VERSIONS, name); + try { + LOG.trace("Entering getKeyVersions method."); + UserGroupInformation user = HttpUserGroupInformation.get(); + KMSClientProvider.checkNotEmpty(name, "name"); + KMSWebApp.getKeyCallsMeter().mark(); + assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_KEY_VERSIONS, name); + LOG.debug("Getting key versions for key {}", name); - List ret = user.doAs( - new PrivilegedExceptionAction>() { - @Override - public List run() throws Exception { - return provider.getKeyVersions(name); - } - } - ); + List ret = user.doAs( + new PrivilegedExceptionAction>() { + @Override + public List run() throws Exception { + return provider.getKeyVersions(name); + } + } + ); - Object json = KMSServerJSONUtils.toJSON(ret); - kmsAudit.ok(user, KMSOp.GET_KEY_VERSIONS, name, ""); - return Response.ok().type(MediaType.APPLICATION_JSON).entity(json).build(); + Object json = KMSServerJSONUtils.toJSON(ret); + kmsAudit.ok(user, KMSOp.GET_KEY_VERSIONS, name, ""); + LOG.trace("Exiting getKeyVersions method."); + return Response.ok().type(MediaType.APPLICATION_JSON).entity(json) + .build(); + } catch (Exception e) { + LOG.debug("Exception in getKeyVersions.", e); + throw e; + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 4c2a967788d..93c0ff09b36 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -2599,8 +2599,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, try (TraceScope ignored = newPathTraceScope("getEZForPath", src)) { return namenode.getEZForPath(src); } catch (RemoteException re) { - throw re.unwrapRemoteException(FileNotFoundException.class, - AccessControlException.class, UnresolvedPathException.class); + throw re.unwrapRemoteException(AccessControlException.class, + UnresolvedPathException.class); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 24ffb40b270..548815fe0a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -41,6 +41,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSLinkResolver; import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemLinkResolver; @@ -2204,6 +2205,35 @@ public class DistributedFileSystem extends FileSystem { return dfs.listEncryptionZones(); } + /* HDFS only */ + public FileEncryptionInfo getFileEncryptionInfo(final Path path) + throws IOException { + Path absF = fixRelativePart(path); + return new FileSystemLinkResolver() { + @Override + public FileEncryptionInfo doCall(final Path p) throws IOException { + final HdfsFileStatus fi = dfs.getFileInfo(getPathName(p)); + if (fi == null) { + throw new FileNotFoundException("File does not exist: " + p); + } + return fi.getFileEncryptionInfo(); + } + + @Override + public FileEncryptionInfo next(final FileSystem fs, final Path p) + throws IOException { + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem myDfs = (DistributedFileSystem)fs; + return myDfs.getFileEncryptionInfo(p); + } + throw new UnsupportedOperationException( + "Cannot call getFileEncryptionInfo" + + " on a symlink to a non-DistributedFileSystem: " + path + + " -> " + p); + } + }.resolve(this, absF); + } + @Override public void setXAttr(Path path, final String name, final byte[] value, final EnumSet flag) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index 946b79dd470..b12fe01b377 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockStoragePolicySpi; import org.apache.hadoop.fs.CacheFlag; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -328,14 +329,13 @@ public class HdfsAdmin { * Get the path of the encryption zone for a given file or directory. * * @param path The path to get the ez for. - * - * @return The EncryptionZone of the ez, or null if path is not in an ez. + * @return An EncryptionZone, or null if path does not exist or is not in an + * ez. * @throws IOException if there was a general IO exception * @throws AccessControlException if the caller does not have access to path - * @throws FileNotFoundException if the path does not exist */ public EncryptionZone getEncryptionZoneForPath(Path path) - throws IOException, AccessControlException, FileNotFoundException { + throws IOException, AccessControlException { return dfs.getEZForPath(path); } @@ -354,6 +354,19 @@ public class HdfsAdmin { return dfs.listEncryptionZones(); } + /** + * Returns the FileEncryptionInfo on the HdfsFileStatus for the given path. + * The return value can be null if the path points to a directory, or a file + * that is not in an encryption zone. + * + * @throws FileNotFoundException if the path does not exist. + * @throws AccessControlException if no execute permission on parent path. + */ + public FileEncryptionInfo getFileEncryptionInfo(final Path path) + throws IOException { + return dfs.getFileEncryptionInfo(path); + } + /** * Exposes a stream of namesystem events. Only events occurring after the * stream is created are available. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index 62ade70442e..bd02a9775c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -26,13 +26,14 @@ import java.nio.MappedByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; -import java.util.TreeMap; +import java.util.NoSuchElementException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import org.apache.commons.collections.map.LinkedMap; import org.apache.commons.lang.mutable.MutableBoolean; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.ExtendedBlockId; @@ -107,16 +108,20 @@ public class ShortCircuitCache implements Closeable { int numDemoted = demoteOldEvictableMmaped(curMs); int numPurged = 0; - Long evictionTimeNs = (long) 0; + Long evictionTimeNs; while (true) { - Entry entry = - evictable.ceilingEntry(evictionTimeNs); - if (entry == null) break; - evictionTimeNs = entry.getKey(); + Object eldestKey; + try { + eldestKey = evictable.firstKey(); + } catch (NoSuchElementException e) { + break; + } + evictionTimeNs = (Long)eldestKey; long evictionTimeMs = TimeUnit.MILLISECONDS.convert(evictionTimeNs, TimeUnit.NANOSECONDS); if (evictionTimeMs + maxNonMmappedEvictableLifespanMs >= curMs) break; - ShortCircuitReplica replica = entry.getValue(); + ShortCircuitReplica replica = (ShortCircuitReplica)evictable.get( + eldestKey); if (LOG.isTraceEnabled()) { LOG.trace("CacheCleaner: purging " + replica + ": " + StringUtils.getStackTrace(Thread.currentThread())); @@ -263,11 +268,11 @@ public class ShortCircuitCache implements Closeable { private CacheCleaner cacheCleaner; /** - * Tree of evictable elements. + * LinkedMap of evictable elements. * * Maps (unique) insertion time in nanoseconds to the element. */ - private final TreeMap evictable = new TreeMap<>(); + private final LinkedMap evictable = new LinkedMap(); /** * Maximum total size of the cache, including both mmapped and @@ -281,12 +286,11 @@ public class ShortCircuitCache implements Closeable { private long maxNonMmappedEvictableLifespanMs; /** - * Tree of mmaped evictable elements. + * LinkedMap of mmaped evictable elements. * * Maps (unique) insertion time in nanoseconds to the element. */ - private final TreeMap evictableMmapped = - new TreeMap<>(); + private final LinkedMap evictableMmapped = new LinkedMap(); /** * Maximum number of mmaped evictable elements. @@ -482,13 +486,16 @@ public class ShortCircuitCache implements Closeable { private int demoteOldEvictableMmaped(long now) { int numDemoted = 0; boolean needMoreSpace = false; - Long evictionTimeNs = (long) 0; + Long evictionTimeNs; while (true) { - Entry entry = - evictableMmapped.ceilingEntry(evictionTimeNs); - if (entry == null) break; - evictionTimeNs = entry.getKey(); + Object eldestKey; + try { + eldestKey = evictableMmapped.firstKey(); + } catch (NoSuchElementException e) { + break; + } + evictionTimeNs = (Long)eldestKey; long evictionTimeMs = TimeUnit.MILLISECONDS.convert(evictionTimeNs, TimeUnit.NANOSECONDS); if (evictionTimeMs + maxEvictableMmapedLifespanMs >= now) { @@ -497,7 +504,8 @@ public class ShortCircuitCache implements Closeable { } needMoreSpace = true; } - ShortCircuitReplica replica = entry.getValue(); + ShortCircuitReplica replica = (ShortCircuitReplica)evictableMmapped.get( + eldestKey); if (LOG.isTraceEnabled()) { String rationale = needMoreSpace ? "because we need more space" : "because it's too old"; @@ -527,10 +535,15 @@ public class ShortCircuitCache implements Closeable { return; } ShortCircuitReplica replica; - if (evictableSize == 0) { - replica = evictableMmapped.firstEntry().getValue(); - } else { - replica = evictable.firstEntry().getValue(); + try { + if (evictableSize == 0) { + replica = (ShortCircuitReplica)evictableMmapped.get(evictableMmapped + .firstKey()); + } else { + replica = (ShortCircuitReplica)evictable.get(evictable.firstKey()); + } + } catch (NoSuchElementException e) { + break; } if (LOG.isTraceEnabled()) { LOG.trace(this + ": trimEvictionMaps is purging " + replica + @@ -573,10 +586,11 @@ public class ShortCircuitCache implements Closeable { * @param map The map to remove it from. */ private void removeEvictable(ShortCircuitReplica replica, - TreeMap map) { + LinkedMap map) { Long evictableTimeNs = replica.getEvictableTimeNs(); Preconditions.checkNotNull(evictableTimeNs); - ShortCircuitReplica removed = map.remove(evictableTimeNs); + ShortCircuitReplica removed = (ShortCircuitReplica)map.remove( + evictableTimeNs); Preconditions.checkState(removed == replica, "failed to make %s unevictable", replica); replica.setEvictableTimeNs(null); @@ -593,7 +607,7 @@ public class ShortCircuitCache implements Closeable { * @param map The map to insert it into. */ private void insertEvictable(Long evictionTimeNs, - ShortCircuitReplica replica, TreeMap map) { + ShortCircuitReplica replica, LinkedMap map) { while (map.containsKey(evictionTimeNs)) { evictionTimeNs++; } @@ -861,14 +875,22 @@ public class ShortCircuitCache implements Closeable { IOUtilsClient.cleanup(LOG, cacheCleaner); // Purge all replicas. while (true) { - Entry entry = evictable.firstEntry(); - if (entry == null) break; - purge(entry.getValue()); + Object eldestKey; + try { + eldestKey = evictable.firstKey(); + } catch (NoSuchElementException e) { + break; + } + purge((ShortCircuitReplica)evictable.get(eldestKey)); } while (true) { - Entry entry = evictableMmapped.firstEntry(); - if (entry == null) break; - purge(entry.getValue()); + Object eldestKey; + try { + eldestKey = evictableMmapped.firstKey(); + } catch (NoSuchElementException e) { + break; + } + purge((ShortCircuitReplica)evictableMmapped.get(eldestKey)); } } finally { lock.unlock(); @@ -909,8 +931,8 @@ public class ShortCircuitCache implements Closeable { void visit(int numOutstandingMmaps, Map replicas, Map failedLoads, - Map evictable, - Map evictableMmapped); + LinkedMap evictable, + LinkedMap evictableMmapped); } @VisibleForTesting // ONLY for testing diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java index 48f202c4f57..130c8fd6cf6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AclPermissionParam.java @@ -20,11 +20,11 @@ package org.apache.hadoop.hdfs.web.resources; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys .DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT; +import java.util.Iterator; import java.util.List; import java.util.regex.Pattern; import org.apache.hadoop.fs.permission.AclEntry; -import org.apache.commons.lang.StringUtils; /** AclPermission parameter. */ public class AclPermissionParam extends StringParam { @@ -63,7 +63,24 @@ public class AclPermissionParam extends StringParam { /** * @return parse {@code aclEntry} and return aclspec */ - private static String parseAclSpec(List aclEntry) { - return StringUtils.join(aclEntry, ","); + private static String parseAclSpec(List aclEntries) { + if (aclEntries == null) { + return null; + } + if (aclEntries.isEmpty()) { + return ""; + } + if (aclEntries.size() == 1) { + AclEntry entry = aclEntries.get(0); + return entry == null ? "" : entry.toStringStable(); + } + StringBuilder sb = new StringBuilder(); + Iterator iter = aclEntries.iterator(); + sb.append(iter.next().toStringStable()); + while (iter.hasNext()) { + AclEntry entry = iter.next(); + sb.append(',').append(entry == null ? "" : entry.toStringStable()); + } + return sb.toString(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index df1d63b2441..0aa5fc143d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -299,7 +299,6 @@ site - true true false ${maven.compile.source} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index 46948f96b15..001bc92b545 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -48,6 +48,9 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import static org.apache.hadoop.hdfs.DFSConfigKeys.HTTPFS_BUFFER_SIZE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.HTTP_BUFFER_SIZE_DEFAULT; + /** * FileSystem operation executors used by {@link HttpFSServer}. */ @@ -462,7 +465,8 @@ public class FSOperations { blockSize = fs.getDefaultBlockSize(path); } FsPermission fsPermission = new FsPermission(permission); - int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); + int bufferSize = fs.getConf().getInt(HTTPFS_BUFFER_SIZE_KEY, + HTTP_BUFFER_SIZE_DEFAULT); OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null); IOUtils.copyBytes(is, os, bufferSize, true); os.close(); @@ -752,7 +756,8 @@ public class FSOperations { */ @Override public InputStream execute(FileSystem fs) throws IOException { - int bufferSize = HttpFSServerWebApp.get().getConfig().getInt("httpfs.buffer.size", 4096); + int bufferSize = HttpFSServerWebApp.get().getConfig().getInt( + HTTPFS_BUFFER_SIZE_KEY, HTTP_BUFFER_SIZE_DEFAULT); return fs.open(path, bufferSize); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java index 0b767bec6d0..61d3b4505fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java @@ -50,6 +50,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; + @InterfaceAudience.Private public class FileSystemAccessService extends BaseService implements FileSystemAccess { private static final Logger LOG = LoggerFactory.getLogger(FileSystemAccessService.class); @@ -159,7 +161,7 @@ public class FileSystemAccessService extends BaseService implements FileSystemAc throw new ServiceException(FileSystemAccessException.ERROR.H01, KERBEROS_PRINCIPAL); } Configuration conf = new Configuration(); - conf.set("hadoop.security.authentication", "kerberos"); + conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); try { UserGroupInformation.loginUserFromKeytab(principal, keytab); @@ -169,7 +171,7 @@ public class FileSystemAccessService extends BaseService implements FileSystemAc LOG.info("Using FileSystemAccess Kerberos authentication, principal [{}] keytab [{}]", principal, keytab); } else if (security.equals("simple")) { Configuration conf = new Configuration(); - conf.set("hadoop.security.authentication", "simple"); + conf.set(HADOOP_SECURITY_AUTHENTICATION, "simple"); UserGroupInformation.setConfiguration(conf); LOG.info("Using FileSystemAccess simple/pseudo authentication, principal [{}]", System.getProperty("user.name")); } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h index c856928c178..83c1c5902a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h @@ -493,6 +493,7 @@ extern "C" { * complete before proceeding with further file updates. * -1 on error. */ + LIBHDFS_EXTERNAL int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength); /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/README.txt b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/README.txt deleted file mode 100644 index 7f672263e7f..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/README.txt +++ /dev/null @@ -1,66 +0,0 @@ -This module provides a BookKeeper backend for HFDS Namenode write -ahead logging. - -BookKeeper is a highly available distributed write ahead logging -system. For more details, see - - http://zookeeper.apache.org/bookkeeper - -------------------------------------------------------------------------------- -How do I build? - - To generate the distribution packages for BK journal, do the - following. - - $ mvn clean package -Pdist - - This will generate a jar with all the dependencies needed by the journal - manager, - - target/hadoop-hdfs-bkjournal-.jar - - Note that the -Pdist part of the build command is important, as otherwise - the dependencies would not be packaged in the jar. - -------------------------------------------------------------------------------- -How do I use the BookKeeper Journal? - - To run a HDFS namenode using BookKeeper as a backend, copy the bkjournal - jar, generated above, into the lib directory of hdfs. In the standard - distribution of HDFS, this is at $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/ - - cp target/hadoop-hdfs-bkjournal-.jar \ - $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/ - - Then, in hdfs-site.xml, set the following properties. - - - dfs.namenode.edits.dir - bookkeeper://localhost:2181/bkjournal,file:///path/for/edits - - - - dfs.namenode.edits.journal-plugin.bookkeeper - org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager - - - In this example, the namenode is configured to use 2 write ahead - logging devices. One writes to BookKeeper and the other to a local - file system. At the moment is is not possible to only write to - BookKeeper, as the resource checker explicitly checked for local - disks currently. - - The given example, configures the namenode to look for the journal - metadata at the path /bkjournal on the a standalone zookeeper ensemble - at localhost:2181. To configure a multiple host zookeeper ensemble, - separate the hosts with semicolons. For example, if you have 3 - zookeeper servers, zk1, zk2 & zk3, each listening on port 2181, you - would specify this with - - bookkeeper://zk1:2181;zk2:2181;zk3:2181/bkjournal - - The final part /bkjournal specifies the znode in zookeeper where - ledger metadata will be store. Administrators can set this to anything - they wish. - - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/dev-support/findbugsExcludeFile.xml deleted file mode 100644 index 45c3a75e1e0..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/dev-support/findbugsExcludeFile.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml deleted file mode 100644 index 7fb6c246dfa..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml +++ /dev/null @@ -1,175 +0,0 @@ - - - - 4.0.0 - - org.apache.hadoop - hadoop-project - 3.0.0-alpha2-SNAPSHOT - ../../../../../hadoop-project - - - org.apache.hadoop.contrib - hadoop-hdfs-bkjournal - 3.0.0-alpha2-SNAPSHOT - Apache Hadoop HDFS BookKeeper Journal - Apache Hadoop HDFS BookKeeper Journal - jar - - - hdfs - ${basedir}/../../../../../hadoop-common-project/hadoop-common/target - - - - - commons-logging - commons-logging - compile - - - org.apache.hadoop - hadoop-common - provided - - - org.apache.hadoop - hadoop-hdfs - provided - - - org.apache.hadoop - hadoop-hdfs - test-jar - test - - - org.apache.hadoop - hadoop-common - test-jar - test - - - org.apache.bookkeeper - bookkeeper-server - compile - - - org.apache.zookeeper - zookeeper - compile - - - com.google.guava - guava - compile - - - junit - junit - test - - - org.mockito - mockito-all - test - - - - - - org.apache.hadoop - hadoop-maven-plugins - - - compile-protoc - generate-sources - - protoc - - - ${protobuf.version} - ${protoc.path} - - ${basedir}/../../../../../hadoop-common-project/hadoop-common/src/main/proto - ${basedir}/../../../../../hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto - ${basedir}/../../../../../hadoop-hdfs-project/hadoop-hdfs/src/main/proto - ${basedir}/src/main/proto - - - ${basedir}/src/main/proto - - bkjournal.proto - - - ${project.build.directory}/generated-sources/java - - - - - - org.codehaus.mojo - findbugs-maven-plugin - - ${basedir}/dev-support/findbugsExcludeFile.xml - - - - org.apache.rat - apache-rat-plugin - - - dev-support/findbugsExcludeFile.xml - - - - - - - - dist - - - - maven-dependency-plugin - 2.8 - - - dist - package - - copy - - - - - org.apache.bookkeeper - bookkeeper-server - jar - - - ${project.build.directory}/lib - - - - - - - - - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java deleted file mode 100644 index 86da80728b6..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java +++ /dev/null @@ -1,264 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Enumeration; - -import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader; -import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.client.LedgerEntry; -import org.apache.bookkeeper.client.BKException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * Input stream which reads from a BookKeeper ledger. - */ -class BookKeeperEditLogInputStream extends EditLogInputStream { - static final Log LOG = LogFactory.getLog(BookKeeperEditLogInputStream.class); - - private final long firstTxId; - private final long lastTxId; - private final int logVersion; - private final boolean inProgress; - private final LedgerHandle lh; - - private final FSEditLogOp.Reader reader; - private final FSEditLogLoader.PositionTrackingInputStream tracker; - - /** - * Construct BookKeeper edit log input stream. - * Starts reading from the first entry of the ledger. - */ - BookKeeperEditLogInputStream(final LedgerHandle lh, - final EditLogLedgerMetadata metadata) - throws IOException { - this(lh, metadata, 0); - } - - /** - * Construct BookKeeper edit log input stream. - * Starts reading from firstBookKeeperEntry. This allows the stream - * to take a shortcut during recovery, as it doesn't have to read - * every edit log transaction to find out what the last one is. - */ - BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata, - long firstBookKeeperEntry) - throws IOException { - this.lh = lh; - this.firstTxId = metadata.getFirstTxId(); - this.lastTxId = metadata.getLastTxId(); - this.logVersion = metadata.getDataLayoutVersion(); - this.inProgress = metadata.isInProgress(); - - if (firstBookKeeperEntry < 0 - || firstBookKeeperEntry > lh.getLastAddConfirmed()) { - throw new IOException("Invalid first bk entry to read: " - + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed()); - } - BufferedInputStream bin = new BufferedInputStream( - new LedgerInputStream(lh, firstBookKeeperEntry)); - tracker = new FSEditLogLoader.PositionTrackingInputStream(bin); - DataInputStream in = new DataInputStream(tracker); - - reader = FSEditLogOp.Reader.create(in, tracker, logVersion); - } - - @Override - public long getFirstTxId() { - return firstTxId; - } - - @Override - public long getLastTxId() { - return lastTxId; - } - - @Override - public int getVersion(boolean verifyVersion) throws IOException { - return logVersion; - } - - @Override - protected FSEditLogOp nextOp() throws IOException { - return reader.readOp(false); - } - - @Override - public void close() throws IOException { - try { - lh.close(); - } catch (BKException e) { - throw new IOException("Exception closing ledger", e); - } catch (InterruptedException e) { - throw new IOException("Interrupted closing ledger", e); - } - } - - @Override - public long getPosition() { - return tracker.getPos(); - } - - @Override - public long length() throws IOException { - return lh.getLength(); - } - - @Override - public String getName() { - return String.format( - "BookKeeperLedger[ledgerId=%d,firstTxId=%d,lastTxId=%d]", lh.getId(), - firstTxId, lastTxId); - } - - @Override - public boolean isInProgress() { - return inProgress; - } - - /** - * Skip forward to specified transaction id. - * Currently we do this by just iterating forward. - * If this proves to be too expensive, this can be reimplemented - * with a binary search over bk entries - */ - public void skipTo(long txId) throws IOException { - long numToSkip = getFirstTxId() - txId; - - FSEditLogOp op = null; - for (long i = 0; i < numToSkip; i++) { - op = readOp(); - } - if (op != null && op.getTransactionId() != txId-1) { - throw new IOException("Corrupt stream, expected txid " - + (txId-1) + ", got " + op.getTransactionId()); - } - } - - @Override - public String toString() { - return ("BookKeeperEditLogInputStream {" + this.getName() + "}"); - } - - @Override - public void setMaxOpSize(int maxOpSize) { - reader.setMaxOpSize(maxOpSize); - } - - @Override - public boolean isLocalLog() { - return false; - } - - /** - * Input stream implementation which can be used by - * FSEditLogOp.Reader - */ - private static class LedgerInputStream extends InputStream { - private long readEntries; - private InputStream entryStream = null; - private final LedgerHandle lh; - private final long maxEntry; - - /** - * Construct ledger input stream - * @param lh the ledger handle to read from - * @param firstBookKeeperEntry ledger entry to start reading from - */ - LedgerInputStream(LedgerHandle lh, long firstBookKeeperEntry) - throws IOException { - this.lh = lh; - readEntries = firstBookKeeperEntry; - - maxEntry = lh.getLastAddConfirmed(); - } - - /** - * Get input stream representing next entry in the - * ledger. - * @return input stream, or null if no more entries - */ - private InputStream nextStream() throws IOException { - try { - if (readEntries > maxEntry) { - return null; - } - Enumeration entries - = lh.readEntries(readEntries, readEntries); - readEntries++; - if (entries.hasMoreElements()) { - LedgerEntry e = entries.nextElement(); - assert !entries.hasMoreElements(); - return e.getEntryInputStream(); - } - } catch (BKException e) { - throw new IOException("Error reading entries from bookkeeper", e); - } catch (InterruptedException e) { - throw new IOException("Interrupted reading entries from bookkeeper", e); - } - return null; - } - - @Override - public int read() throws IOException { - byte[] b = new byte[1]; - if (read(b, 0, 1) != 1) { - return -1; - } else { - return b[0]; - } - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - try { - int read = 0; - if (entryStream == null) { - entryStream = nextStream(); - if (entryStream == null) { - return read; - } - } - - while (read < len) { - int thisread = entryStream.read(b, off+read, (len-read)); - if (thisread == -1) { - entryStream = nextStream(); - if (entryStream == null) { - return read; - } - } else { - read += thisread; - } - } - return read; - } catch (IOException e) { - throw e; - } - - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java deleted file mode 100644 index 865806b69c9..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogOutputStream.java +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.CountDownLatch; - -import java.util.Arrays; - -import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.client.BKException; -import org.apache.bookkeeper.client.AsyncCallback.AddCallback; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.Writer; - -import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; -import org.apache.hadoop.io.DataOutputBuffer; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * Output stream for BookKeeper Journal. - * Multiple complete edit log entries are packed into a single bookkeeper - * entry before sending it over the network. The fact that the edit log entries - * are complete in the bookkeeper entries means that each bookkeeper log entry - *can be read as a complete edit log. This is useful for recover, as we don't - * need to read through the entire edit log segment to get the last written - * entry. - */ -class BookKeeperEditLogOutputStream - extends EditLogOutputStream implements AddCallback { - static final Log LOG = LogFactory.getLog(BookKeeperEditLogOutputStream.class); - - private final DataOutputBuffer bufCurrent; - private final AtomicInteger outstandingRequests; - private final int transmissionThreshold; - private final LedgerHandle lh; - private CountDownLatch syncLatch; - private final AtomicInteger transmitResult - = new AtomicInteger(BKException.Code.OK); - private final Writer writer; - - /** - * Construct an edit log output stream which writes to a ledger. - - */ - protected BookKeeperEditLogOutputStream(Configuration conf, LedgerHandle lh) - throws IOException { - super(); - - bufCurrent = new DataOutputBuffer(); - outstandingRequests = new AtomicInteger(0); - syncLatch = null; - this.lh = lh; - this.writer = new Writer(bufCurrent); - this.transmissionThreshold - = conf.getInt(BookKeeperJournalManager.BKJM_OUTPUT_BUFFER_SIZE, - BookKeeperJournalManager.BKJM_OUTPUT_BUFFER_SIZE_DEFAULT); - } - - @Override - public void create(int layoutVersion) throws IOException { - // noop - } - - @Override - public void close() throws IOException { - setReadyToFlush(); - flushAndSync(true); - try { - lh.close(); - } catch (InterruptedException ie) { - throw new IOException("Interrupted waiting on close", ie); - } catch (BKException bke) { - throw new IOException("BookKeeper error during close", bke); - } - } - - @Override - public void abort() throws IOException { - try { - lh.close(); - } catch (InterruptedException ie) { - throw new IOException("Interrupted waiting on close", ie); - } catch (BKException bke) { - throw new IOException("BookKeeper error during abort", bke); - } - - } - - @Override - public void writeRaw(final byte[] data, int off, int len) throws IOException { - throw new IOException("Not supported for BK"); - } - - @Override - public void write(FSEditLogOp op) throws IOException { - writer.writeOp(op); - - if (bufCurrent.getLength() > transmissionThreshold) { - transmit(); - } - } - - @Override - public void setReadyToFlush() throws IOException { - transmit(); - - synchronized (this) { - syncLatch = new CountDownLatch(outstandingRequests.get()); - } - } - - @Override - public void flushAndSync(boolean durable) throws IOException { - assert(syncLatch != null); - try { - syncLatch.await(); - } catch (InterruptedException ie) { - throw new IOException("Interrupted waiting on latch", ie); - } - if (transmitResult.get() != BKException.Code.OK) { - throw new IOException("Failed to write to bookkeeper; Error is (" - + transmitResult.get() + ") " - + BKException.getMessage(transmitResult.get())); - } - - syncLatch = null; - // wait for whatever we wait on - } - - /** - * Transmit the current buffer to bookkeeper. - * Synchronised at the FSEditLog level. #write() and #setReadyToFlush() - * are never called at the same time. - */ - private void transmit() throws IOException { - if (!transmitResult.compareAndSet(BKException.Code.OK, - BKException.Code.OK)) { - throw new IOException("Trying to write to an errored stream;" - + " Error code : (" + transmitResult.get() - + ") " + BKException.getMessage(transmitResult.get())); - } - if (bufCurrent.getLength() > 0) { - byte[] entry = Arrays.copyOf(bufCurrent.getData(), - bufCurrent.getLength()); - lh.asyncAddEntry(entry, this, null); - bufCurrent.reset(); - outstandingRequests.incrementAndGet(); - } - } - - @Override - public void addComplete(int rc, LedgerHandle handle, - long entryId, Object ctx) { - synchronized(this) { - outstandingRequests.decrementAndGet(); - if (!transmitResult.compareAndSet(BKException.Code.OK, rc)) { - LOG.warn("Tried to set transmit result to (" + rc + ") \"" - + BKException.getMessage(rc) + "\"" - + " but is already (" + transmitResult.get() + ") \"" - + BKException.getMessage(transmitResult.get()) + "\""); - } - CountDownLatch l = syncLatch; - if (l != null) { - l.countDown(); - } - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java deleted file mode 100644 index 8e4d032691b..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java +++ /dev/null @@ -1,893 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.common.Storage; -import org.apache.hadoop.hdfs.server.common.StorageInfo; -import org.apache.hadoop.hdfs.server.namenode.JournalManager; -import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; -import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; -import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import org.apache.hadoop.conf.Configuration; - -import org.apache.bookkeeper.conf.ClientConfiguration; -import org.apache.bookkeeper.client.BKException; -import org.apache.bookkeeper.client.BookKeeper; -import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.util.ZkUtils; - -import org.apache.zookeeper.data.Stat; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.AsyncCallback.StringCallback; -import org.apache.zookeeper.ZKUtil; - -import java.util.Collection; -import java.util.Collections; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.io.IOException; - -import java.net.URI; - -import org.apache.hadoop.hdfs.protocolPB.PBHelper; -import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.VersionProto; -import com.google.protobuf.TextFormat; -import static com.google.common.base.Charsets.UTF_8; - -import org.apache.commons.io.Charsets; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import com.google.common.annotations.VisibleForTesting; -/** - * BookKeeper Journal Manager - * - * To use, add the following to hdfs-site.xml. - * 

    - * {@code
- * 
- *   dfs.namenode.edits.dir
- *   bookkeeper://zk1:2181;zk2:2181;zk3:2181/hdfsjournal
- * 
- *
- * 
- *   dfs.namenode.edits.journal-plugin.bookkeeper
- *   org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager
- * 
- * }
- *
    - * The URI format for bookkeeper is bookkeeper://[zkEnsemble]/[rootZnode] - * [zookkeeper ensemble] is a list of semi-colon separated, zookeeper host:port - * pairs. In the example above there are 3 servers, in the ensemble, - * zk1, zk2 & zk3, each one listening on port 2181. - * - * [root znode] is the path of the zookeeper znode, under which the editlog - * information will be stored. - * - * Other configuration options are: - *
      - *
    • dfs.namenode.bookkeeperjournal.output-buffer-size - * Number of bytes a bookkeeper journal stream will buffer before - * forcing a flush. Default is 1024.
      • - *
    • dfs.namenode.bookkeeperjournal.ensemble-size - * Number of bookkeeper servers in edit log ledger ensembles. This - * is the number of bookkeeper servers which need to be available - * for the ledger to be writable. Default is 3.
      • - *
    • dfs.namenode.bookkeeperjournal.quorum-size - * Number of bookkeeper servers in the write quorum. This is the - * number of bookkeeper servers which must have acknowledged the - * write of an entry before it is considered written. - * Default is 2.
      • - *
    • dfs.namenode.bookkeeperjournal.digestPw - * Password to use when creating ledgers.
      • - *
    • dfs.namenode.bookkeeperjournal.zk.session.timeout - * Session timeout for Zookeeper client from BookKeeper Journal Manager. - * Hadoop recommends that, this value should be less than the ZKFC - * session timeout value. Default value is 3000.
      • - *
    - */ -public class BookKeeperJournalManager implements JournalManager { - static final Log LOG = LogFactory.getLog(BookKeeperJournalManager.class); - - public static final String BKJM_OUTPUT_BUFFER_SIZE - = "dfs.namenode.bookkeeperjournal.output-buffer-size"; - public static final int BKJM_OUTPUT_BUFFER_SIZE_DEFAULT = 1024; - - public static final String BKJM_BOOKKEEPER_ENSEMBLE_SIZE - = "dfs.namenode.bookkeeperjournal.ensemble-size"; - public static final int BKJM_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT = 3; - - public static final String BKJM_BOOKKEEPER_QUORUM_SIZE - = "dfs.namenode.bookkeeperjournal.quorum-size"; - public static final int BKJM_BOOKKEEPER_QUORUM_SIZE_DEFAULT = 2; - - public static final String BKJM_BOOKKEEPER_DIGEST_PW - = "dfs.namenode.bookkeeperjournal.digestPw"; - public static final String BKJM_BOOKKEEPER_DIGEST_PW_DEFAULT = ""; - - private static final int BKJM_LAYOUT_VERSION = -1; - - public static final String BKJM_ZK_SESSION_TIMEOUT - = "dfs.namenode.bookkeeperjournal.zk.session.timeout"; - public static final int BKJM_ZK_SESSION_TIMEOUT_DEFAULT = 3000; - - private static final String BKJM_EDIT_INPROGRESS = "inprogress_"; - - public static final String BKJM_ZK_LEDGERS_AVAILABLE_PATH - = "dfs.namenode.bookkeeperjournal.zk.availablebookies"; - - public static final String BKJM_ZK_LEDGERS_AVAILABLE_PATH_DEFAULT - = "/ledgers/available"; - - public static final String BKJM_BOOKKEEPER_SPECULATIVE_READ_TIMEOUT_MS - = "dfs.namenode.bookkeeperjournal.speculativeReadTimeoutMs"; - public static final int BKJM_BOOKKEEPER_SPECULATIVE_READ_TIMEOUT_DEFAULT - = 2000; - - public static final String BKJM_BOOKKEEPER_READ_ENTRY_TIMEOUT_SEC - = "dfs.namenode.bookkeeperjournal.readEntryTimeoutSec"; - public static final int BKJM_BOOKKEEPER_READ_ENTRY_TIMEOUT_DEFAULT = 5; - - public static final String BKJM_BOOKKEEPER_ACK_QUORUM_SIZE - = "dfs.namenode.bookkeeperjournal.ack.quorum-size"; - - public static final String BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_SEC - = "dfs.namenode.bookkeeperjournal.addEntryTimeoutSec"; - public static final int BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_DEFAULT = 5; - - private ZooKeeper zkc; - private final Configuration conf; - private final BookKeeper bkc; - private final CurrentInprogress ci; - private final String basePath; - private final String ledgerPath; - private final String versionPath; - private final MaxTxId maxTxId; - private final int ensembleSize; - private final int quorumSize; - private final int ackQuorumSize; - private final int addEntryTimeout; - private final String digestpw; - private final int speculativeReadTimeout; - private final int readEntryTimeout; - private final CountDownLatch zkConnectLatch; - private final NamespaceInfo nsInfo; - private boolean initialized = false; - private LedgerHandle currentLedger = null; - - /** - * Construct a Bookkeeper journal manager. - */ - public BookKeeperJournalManager(Configuration conf, URI uri, - NamespaceInfo nsInfo) throws IOException { - this.conf = conf; - this.nsInfo = nsInfo; - - String zkConnect = uri.getAuthority().replace(";", ","); - basePath = uri.getPath(); - ensembleSize = conf.getInt(BKJM_BOOKKEEPER_ENSEMBLE_SIZE, - BKJM_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT); - quorumSize = conf.getInt(BKJM_BOOKKEEPER_QUORUM_SIZE, - BKJM_BOOKKEEPER_QUORUM_SIZE_DEFAULT); - ackQuorumSize = conf.getInt(BKJM_BOOKKEEPER_ACK_QUORUM_SIZE, quorumSize); - addEntryTimeout = conf.getInt(BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_SEC, - BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_DEFAULT); - speculativeReadTimeout = conf.getInt( - BKJM_BOOKKEEPER_SPECULATIVE_READ_TIMEOUT_MS, - BKJM_BOOKKEEPER_SPECULATIVE_READ_TIMEOUT_DEFAULT); - readEntryTimeout = conf.getInt(BKJM_BOOKKEEPER_READ_ENTRY_TIMEOUT_SEC, - BKJM_BOOKKEEPER_READ_ENTRY_TIMEOUT_DEFAULT); - - ledgerPath = basePath + "/ledgers"; - String maxTxIdPath = basePath + "/maxtxid"; - String currentInprogressNodePath = basePath + "/CurrentInprogress"; - versionPath = basePath + "/version"; - digestpw = conf.get(BKJM_BOOKKEEPER_DIGEST_PW, - BKJM_BOOKKEEPER_DIGEST_PW_DEFAULT); - - try { - zkConnectLatch = new CountDownLatch(1); - int bkjmZKSessionTimeout = conf.getInt(BKJM_ZK_SESSION_TIMEOUT, - BKJM_ZK_SESSION_TIMEOUT_DEFAULT); - zkc = new ZooKeeper(zkConnect, bkjmZKSessionTimeout, - new ZkConnectionWatcher()); - // Configured zk session timeout + some extra grace period (here - // BKJM_ZK_SESSION_TIMEOUT_DEFAULT used as grace period) - int zkConnectionLatchTimeout = bkjmZKSessionTimeout - + BKJM_ZK_SESSION_TIMEOUT_DEFAULT; - if (!zkConnectLatch - .await(zkConnectionLatchTimeout, TimeUnit.MILLISECONDS)) { - throw new IOException("Error connecting to zookeeper"); - } - - prepareBookKeeperEnv(); - ClientConfiguration clientConf = new ClientConfiguration(); - clientConf.setSpeculativeReadTimeout(speculativeReadTimeout); - clientConf.setReadEntryTimeout(readEntryTimeout); - clientConf.setAddEntryTimeout(addEntryTimeout); - bkc = new BookKeeper(clientConf, zkc); - } catch (KeeperException e) { - throw new IOException("Error initializing zk", e); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted while initializing bk journal manager", - ie); - } - - ci = new CurrentInprogress(zkc, currentInprogressNodePath); - maxTxId = new MaxTxId(zkc, maxTxIdPath); - } - - /** - * Pre-creating bookkeeper metadata path in zookeeper. - */ - private void prepareBookKeeperEnv() throws IOException { - // create bookie available path in zookeeper if it doesn't exists - final String zkAvailablePath = conf.get(BKJM_ZK_LEDGERS_AVAILABLE_PATH, - BKJM_ZK_LEDGERS_AVAILABLE_PATH_DEFAULT); - final CountDownLatch zkPathLatch = new CountDownLatch(1); - - final AtomicBoolean success = new AtomicBoolean(false); - StringCallback callback = new StringCallback() { - @Override - public void processResult(int rc, String path, Object ctx, String name) { - if (KeeperException.Code.OK.intValue() == rc - || KeeperException.Code.NODEEXISTS.intValue() == rc) { - LOG.info("Successfully created bookie available path : " - + zkAvailablePath); - success.set(true); - } else { - KeeperException.Code code = KeeperException.Code.get(rc); - LOG.error("Error : " - + KeeperException.create(code, path).getMessage() - + ", failed to create bookie available path : " - + zkAvailablePath); - } - zkPathLatch.countDown(); - } - }; - ZkUtils.asyncCreateFullPathOptimistic(zkc, zkAvailablePath, new byte[0], - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, callback, null); - - try { - if (!zkPathLatch.await(zkc.getSessionTimeout(), TimeUnit.MILLISECONDS) - || !success.get()) { - throw new IOException("Couldn't create bookie available path :" - + zkAvailablePath + ", timed out " + zkc.getSessionTimeout() - + " millis"); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException( - "Interrupted when creating the bookie available path : " - + zkAvailablePath, e); - } - } - - @Override - public void format(NamespaceInfo ns) throws IOException { - try { - // delete old info - Stat baseStat = null; - Stat ledgerStat = null; - if ((baseStat = zkc.exists(basePath, false)) != null) { - if ((ledgerStat = zkc.exists(ledgerPath, false)) != null) { - for (EditLogLedgerMetadata l : getLedgerList(true)) { - try { - bkc.deleteLedger(l.getLedgerId()); - } catch (BKException.BKNoSuchLedgerExistsException bke) { - LOG.warn("Ledger " + l.getLedgerId() + " does not exist;" - + " Cannot delete."); - } - } - } - ZKUtil.deleteRecursive(zkc, basePath); - } - - // should be clean now. - zkc.create(basePath, new byte[] {'0'}, - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - - VersionProto.Builder builder = VersionProto.newBuilder(); - builder.setNamespaceInfo(PBHelper.convert(ns)) - .setLayoutVersion(BKJM_LAYOUT_VERSION); - - byte[] data = TextFormat.printToString(builder.build()).getBytes(UTF_8); - zkc.create(versionPath, data, - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - - zkc.create(ledgerPath, new byte[] {'0'}, - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } catch (KeeperException ke) { - LOG.error("Error accessing zookeeper to format", ke); - throw new IOException("Error accessing zookeeper to format", ke); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted during format", ie); - } catch (BKException bke) { - throw new IOException("Error cleaning up ledgers during format", bke); - } - } - - @Override - public boolean hasSomeData() throws IOException { - try { - return zkc.exists(basePath, false) != null; - } catch (KeeperException ke) { - throw new IOException("Couldn't contact zookeeper", ke); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted while checking for data", ie); - } - } - - synchronized private void checkEnv() throws IOException { - if (!initialized) { - try { - Stat versionStat = zkc.exists(versionPath, false); - if (versionStat == null) { - throw new IOException("Environment not initialized. " - +"Have you forgotten to format?"); - } - byte[] d = zkc.getData(versionPath, false, versionStat); - - VersionProto.Builder builder = VersionProto.newBuilder(); - TextFormat.merge(new String(d, UTF_8), builder); - if (!builder.isInitialized()) { - throw new IOException("Invalid/Incomplete data in znode"); - } - VersionProto vp = builder.build(); - - // There's only one version at the moment - assert vp.getLayoutVersion() == BKJM_LAYOUT_VERSION; - - NamespaceInfo readns = PBHelper.convert(vp.getNamespaceInfo()); - - if (nsInfo.getNamespaceID() != readns.getNamespaceID() || - !nsInfo.clusterID.equals(readns.getClusterID()) || - !nsInfo.getBlockPoolID().equals(readns.getBlockPoolID())) { - String err = String.format("Environment mismatch. Running process %s" - +", stored in ZK %s", nsInfo, readns); - LOG.error(err); - throw new IOException(err); - } - - ci.init(); - initialized = true; - } catch (KeeperException ke) { - throw new IOException("Cannot access ZooKeeper", ke); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted while checking environment", ie); - } - } - } - - /** - * Start a new log segment in a BookKeeper ledger. - * First ensure that we have the write lock for this journal. - * Then create a ledger and stream based on that ledger. - * The ledger id is written to the inprogress znode, so that in the - * case of a crash, a recovery process can find the ledger we were writing - * to when we crashed. - * @param txId First transaction id to be written to the stream - */ - @Override - public EditLogOutputStream startLogSegment(long txId, int layoutVersion) - throws IOException { - checkEnv(); - - if (txId <= maxTxId.get()) { - throw new IOException("We've already seen " + txId - + ". A new stream cannot be created with it"); - } - - try { - String existingInprogressNode = ci.read(); - if (null != existingInprogressNode - && zkc.exists(existingInprogressNode, false) != null) { - throw new IOException("Inprogress node already exists"); - } - if (currentLedger != null) { - // bookkeeper errored on last stream, clean up ledger - currentLedger.close(); - } - currentLedger = bkc.createLedger(ensembleSize, quorumSize, ackQuorumSize, - BookKeeper.DigestType.MAC, - digestpw.getBytes(Charsets.UTF_8)); - } catch (BKException bke) { - throw new IOException("Error creating ledger", bke); - } catch (KeeperException ke) { - throw new IOException("Error in zookeeper while creating ledger", ke); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted creating ledger", ie); - } - - try { - String znodePath = inprogressZNode(txId); - EditLogLedgerMetadata l = new EditLogLedgerMetadata(znodePath, - layoutVersion, currentLedger.getId(), txId); - /* Write the ledger metadata out to the inprogress ledger znode - * This can fail if for some reason our write lock has - * expired (@see WriteLock) and another process has managed to - * create the inprogress znode. - * In this case, throw an exception. We don't want to continue - * as this would lead to a split brain situation. - */ - l.write(zkc, znodePath); - - maxTxId.store(txId); - ci.update(znodePath); - return new BookKeeperEditLogOutputStream(conf, currentLedger); - } catch (KeeperException ke) { - cleanupLedger(currentLedger); - throw new IOException("Error storing ledger metadata", ke); - } - } - - private void cleanupLedger(LedgerHandle lh) { - try { - long id = currentLedger.getId(); - currentLedger.close(); - bkc.deleteLedger(id); - } catch (BKException bke) { - //log & ignore, an IOException will be thrown soon - LOG.error("Error closing ledger", bke); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - LOG.warn("Interrupted while closing ledger", ie); - } - } - - - - /** - * Finalize a log segment. If the journal manager is currently - * writing to a ledger, ensure that this is the ledger of the log segment - * being finalized. - * - * Otherwise this is the recovery case. In the recovery case, ensure that - * the firstTxId of the ledger matches firstTxId for the segment we are - * trying to finalize. - */ - @Override - public void finalizeLogSegment(long firstTxId, long lastTxId) - throws IOException { - checkEnv(); - - String inprogressPath = inprogressZNode(firstTxId); - try { - Stat inprogressStat = zkc.exists(inprogressPath, false); - if (inprogressStat == null) { - throw new IOException("Inprogress znode " + inprogressPath - + " doesn't exist"); - } - - EditLogLedgerMetadata l - = EditLogLedgerMetadata.read(zkc, inprogressPath); - - if (currentLedger != null) { // normal, non-recovery case - if (l.getLedgerId() == currentLedger.getId()) { - try { - currentLedger.close(); - } catch (BKException bke) { - LOG.error("Error closing current ledger", bke); - } - currentLedger = null; - } else { - throw new IOException( - "Active ledger has different ID to inprogress. " - + l.getLedgerId() + " found, " - + currentLedger.getId() + " expected"); - } - } - - if (l.getFirstTxId() != firstTxId) { - throw new IOException("Transaction id not as expected, " - + l.getFirstTxId() + " found, " + firstTxId + " expected"); - } - - l.finalizeLedger(lastTxId); - String finalisedPath = finalizedLedgerZNode(firstTxId, lastTxId); - try { - l.write(zkc, finalisedPath); - } catch (KeeperException.NodeExistsException nee) { - if (!l.verify(zkc, finalisedPath)) { - throw new IOException("Node " + finalisedPath + " already exists" - + " but data doesn't match"); - } - } - maxTxId.store(lastTxId); - zkc.delete(inprogressPath, inprogressStat.getVersion()); - String inprogressPathFromCI = ci.read(); - if (inprogressPath.equals(inprogressPathFromCI)) { - ci.clear(); - } - } catch (KeeperException e) { - throw new IOException("Error finalising ledger", e); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IOException("Error finalising ledger", ie); - } - } - - public void selectInputStreams( - Collection streams, - long fromTxnId, boolean inProgressOk) throws IOException { - selectInputStreams(streams, fromTxnId, inProgressOk, false); - } - - @Override - public void selectInputStreams(Collection streams, - long fromTxId, boolean inProgressOk, boolean onlyDurableTxns) - throws IOException { - List currentLedgerList = getLedgerList(fromTxId, - inProgressOk); - try { - BookKeeperEditLogInputStream elis = null; - for (EditLogLedgerMetadata l : currentLedgerList) { - long lastTxId = l.getLastTxId(); - if (l.isInProgress()) { - lastTxId = recoverLastTxId(l, false); - } - // Check once again, required in case of InProgress and is case of any - // gap. - if (fromTxId >= l.getFirstTxId() && fromTxId <= lastTxId) { - LedgerHandle h; - if (l.isInProgress()) { // we don't want to fence the current journal - h = bkc.openLedgerNoRecovery(l.getLedgerId(), - BookKeeper.DigestType.MAC, digestpw.getBytes(Charsets.UTF_8)); - } else { - h = bkc.openLedger(l.getLedgerId(), BookKeeper.DigestType.MAC, - digestpw.getBytes(Charsets.UTF_8)); - } - elis = new BookKeeperEditLogInputStream(h, l); - elis.skipTo(fromTxId); - } else { - // If mismatches then there might be some gap, so we should not check - // further. - return; - } - streams.add(elis); - if (elis.getLastTxId() == HdfsServerConstants.INVALID_TXID) { - return; - } - fromTxId = elis.getLastTxId() + 1; - } - } catch (BKException e) { - throw new IOException("Could not open ledger for " + fromTxId, e); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted opening ledger for " + fromTxId, ie); - } - } - - long getNumberOfTransactions(long fromTxId, boolean inProgressOk) - throws IOException { - long count = 0; - long expectedStart = 0; - for (EditLogLedgerMetadata l : getLedgerList(inProgressOk)) { - long lastTxId = l.getLastTxId(); - if (l.isInProgress()) { - lastTxId = recoverLastTxId(l, false); - if (lastTxId == HdfsServerConstants.INVALID_TXID) { - break; - } - } - - assert lastTxId >= l.getFirstTxId(); - - if (lastTxId < fromTxId) { - continue; - } else if (l.getFirstTxId() <= fromTxId && lastTxId >= fromTxId) { - // we can start in the middle of a segment - count = (lastTxId - l.getFirstTxId()) + 1; - expectedStart = lastTxId + 1; - } else { - if (expectedStart != l.getFirstTxId()) { - if (count == 0) { - throw new CorruptionException("StartTxId " + l.getFirstTxId() - + " is not as expected " + expectedStart - + ". Gap in transaction log?"); - } else { - break; - } - } - count += (lastTxId - l.getFirstTxId()) + 1; - expectedStart = lastTxId + 1; - } - } - return count; - } - - @Override - public void recoverUnfinalizedSegments() throws IOException { - checkEnv(); - - synchronized (this) { - try { - List children = zkc.getChildren(ledgerPath, false); - for (String child : children) { - if (!child.startsWith(BKJM_EDIT_INPROGRESS)) { - continue; - } - String znode = ledgerPath + "/" + child; - EditLogLedgerMetadata l = EditLogLedgerMetadata.read(zkc, znode); - try { - long endTxId = recoverLastTxId(l, true); - if (endTxId == HdfsServerConstants.INVALID_TXID) { - LOG.error("Unrecoverable corruption has occurred in segment " - + l.toString() + " at path " + znode - + ". Unable to continue recovery."); - throw new IOException("Unrecoverable corruption," - + " please check logs."); - } - finalizeLogSegment(l.getFirstTxId(), endTxId); - } catch (SegmentEmptyException see) { - LOG.warn("Inprogress znode " + child - + " refers to a ledger which is empty. This occurs when the NN" - + " crashes after opening a segment, but before writing the" - + " OP_START_LOG_SEGMENT op. It is safe to delete." - + " MetaData [" + l.toString() + "]"); - - // If the max seen transaction is the same as what would - // have been the first transaction of the failed ledger, - // decrement it, as that transaction never happened and as - // such, is _not_ the last seen - if (maxTxId.get() == l.getFirstTxId()) { - maxTxId.reset(maxTxId.get() - 1); - } - - zkc.delete(znode, -1); - } - } - } catch (KeeperException.NoNodeException nne) { - // nothing to recover, ignore - } catch (KeeperException ke) { - throw new IOException("Couldn't get list of inprogress segments", ke); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted getting list of inprogress segments", - ie); - } - } - } - - @Override - public void purgeLogsOlderThan(long minTxIdToKeep) - throws IOException { - checkEnv(); - - for (EditLogLedgerMetadata l : getLedgerList(false)) { - if (l.getLastTxId() < minTxIdToKeep) { - try { - Stat stat = zkc.exists(l.getZkPath(), false); - zkc.delete(l.getZkPath(), stat.getVersion()); - bkc.deleteLedger(l.getLedgerId()); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - LOG.error("Interrupted while purging " + l, ie); - } catch (BKException bke) { - LOG.error("Couldn't delete ledger from bookkeeper", bke); - } catch (KeeperException ke) { - LOG.error("Error deleting ledger entry in zookeeper", ke); - } - } - } - } - - @Override - public void doPreUpgrade() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public void doUpgrade(Storage storage) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public long getJournalCTime() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public void doFinalize() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public boolean canRollBack(StorageInfo storage, StorageInfo prevStorage, - int targetLayoutVersion) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public void doRollback() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public void discardSegments(long startTxId) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public void close() throws IOException { - try { - bkc.close(); - zkc.close(); - } catch (BKException bke) { - throw new IOException("Couldn't close bookkeeper client", bke); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted while closing journal manager", ie); - } - } - - /** - * Set the amount of memory that this stream should use to buffer edits. - * Setting this will only affect future output stream. Streams - * which have currently be created won't be affected. - */ - @Override - public void setOutputBufferCapacity(int size) { - conf.getInt(BKJM_OUTPUT_BUFFER_SIZE, size); - } - - /** - * Find the id of the last edit log transaction writen to a edit log - * ledger. - */ - private long recoverLastTxId(EditLogLedgerMetadata l, boolean fence) - throws IOException, SegmentEmptyException { - LedgerHandle lh = null; - try { - if (fence) { - lh = bkc.openLedger(l.getLedgerId(), - BookKeeper.DigestType.MAC, - digestpw.getBytes(Charsets.UTF_8)); - } else { - lh = bkc.openLedgerNoRecovery(l.getLedgerId(), - BookKeeper.DigestType.MAC, - digestpw.getBytes(Charsets.UTF_8)); - } - } catch (BKException bke) { - throw new IOException("Exception opening ledger for " + l, bke); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted opening ledger for " + l, ie); - } - - BookKeeperEditLogInputStream in = null; - - try { - long lastAddConfirmed = lh.getLastAddConfirmed(); - if (lastAddConfirmed == -1) { - throw new SegmentEmptyException(); - } - - in = new BookKeeperEditLogInputStream(lh, l, lastAddConfirmed); - - long endTxId = HdfsServerConstants.INVALID_TXID; - FSEditLogOp op = in.readOp(); - while (op != null) { - if (endTxId == HdfsServerConstants.INVALID_TXID - || op.getTransactionId() == endTxId+1) { - endTxId = op.getTransactionId(); - } - op = in.readOp(); - } - return endTxId; - } finally { - if (in != null) { - in.close(); - } - } - } - - /** - * Get a list of all segments in the journal. - */ - List getLedgerList(boolean inProgressOk) - throws IOException { - return getLedgerList(-1, inProgressOk); - } - - private List getLedgerList(long fromTxId, - boolean inProgressOk) throws IOException { - List ledgers - = new ArrayList(); - try { - List ledgerNames = zkc.getChildren(ledgerPath, false); - for (String ledgerName : ledgerNames) { - if (!inProgressOk && ledgerName.contains(BKJM_EDIT_INPROGRESS)) { - continue; - } - String legderMetadataPath = ledgerPath + "/" + ledgerName; - try { - EditLogLedgerMetadata editLogLedgerMetadata = EditLogLedgerMetadata - .read(zkc, legderMetadataPath); - if (editLogLedgerMetadata.getLastTxId() != HdfsServerConstants.INVALID_TXID - && editLogLedgerMetadata.getLastTxId() < fromTxId) { - // exclude already read closed edits, but include inprogress edits - // as this will be handled in caller - continue; - } - ledgers.add(editLogLedgerMetadata); - } catch (KeeperException.NoNodeException e) { - LOG.warn("ZNode: " + legderMetadataPath - + " might have finalized and deleted." - + " So ignoring NoNodeException."); - } - } - } catch (KeeperException e) { - throw new IOException("Exception reading ledger list from zk", e); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted getting list of ledgers from zk", ie); - } - - Collections.sort(ledgers, EditLogLedgerMetadata.COMPARATOR); - return ledgers; - } - - /** - * Get the znode path for a finalize ledger - */ - String finalizedLedgerZNode(long startTxId, long endTxId) { - return String.format("%s/edits_%018d_%018d", - ledgerPath, startTxId, endTxId); - } - - /** - * Get the znode path for the inprogressZNode - */ - String inprogressZNode(long startTxid) { - return ledgerPath + "/inprogress_" + Long.toString(startTxid, 16); - } - - @VisibleForTesting - void setZooKeeper(ZooKeeper zk) { - this.zkc = zk; - } - - /** - * Simple watcher to notify when zookeeper has connected - */ - private class ZkConnectionWatcher implements Watcher { - public void process(WatchedEvent event) { - if (Event.KeeperState.SyncConnected.equals(event.getState())) { - zkConnectLatch.countDown(); - } - } - } - - private static class SegmentEmptyException extends IOException { - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java deleted file mode 100644 index 32d65cbf8d7..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import java.io.IOException; -import java.net.InetAddress; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.KeeperException.NodeExistsException; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.data.Stat; - -import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.CurrentInprogressProto; -import com.google.protobuf.TextFormat; -import static com.google.common.base.Charsets.UTF_8; - -/** - * Distributed write permission lock, using ZooKeeper. Read the version number - * and return the current inprogress node path available in CurrentInprogress - * path. If it exist, caller can treat that some other client already operating - * on it. Then caller can take action. If there is no inprogress node exist, - * then caller can treat that there is no client operating on it. Later same - * caller should update the his newly created inprogress node path. At this - * point, if some other activities done on this node, version number might - * change, so update will fail. So, this read, update api will ensure that there - * is only node can continue further after checking with CurrentInprogress. - */ - -class CurrentInprogress { - static final Log LOG = LogFactory.getLog(CurrentInprogress.class); - - private final ZooKeeper zkc; - private final String currentInprogressNode; - private volatile int versionNumberForPermission = -1; - private final String hostName = InetAddress.getLocalHost().toString(); - - CurrentInprogress(ZooKeeper zkc, String lockpath) throws IOException { - this.currentInprogressNode = lockpath; - this.zkc = zkc; - } - - void init() throws IOException { - try { - Stat isCurrentInprogressNodeExists = zkc.exists(currentInprogressNode, - false); - if (isCurrentInprogressNodeExists == null) { - try { - zkc.create(currentInprogressNode, null, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - } catch (NodeExistsException e) { - // Node might created by other process at the same time. Ignore it. - if (LOG.isDebugEnabled()) { - LOG.debug(currentInprogressNode + " already created by other process.", - e); - } - } - } - } catch (KeeperException e) { - throw new IOException("Exception accessing Zookeeper", e); - } catch (InterruptedException ie) { - throw new IOException("Interrupted accessing Zookeeper", ie); - } - } - - /** - * Update the path with prepending version number and hostname - * - * @param path - * - to be updated in zookeeper - * @throws IOException - */ - void update(String path) throws IOException { - CurrentInprogressProto.Builder builder = CurrentInprogressProto.newBuilder(); - builder.setPath(path).setHostname(hostName); - - String content = TextFormat.printToString(builder.build()); - - try { - zkc.setData(this.currentInprogressNode, content.getBytes(UTF_8), - this.versionNumberForPermission); - } catch (KeeperException e) { - throw new IOException("Exception when setting the data " - + "[" + content + "] to CurrentInprogress. ", e); - } catch (InterruptedException e) { - throw new IOException("Interrupted while setting the data " - + "[" + content + "] to CurrentInprogress", e); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Updated data[" + content + "] to CurrentInprogress"); - } - } - - /** - * Read the CurrentInprogress node data from Zookeeper and also get the znode - * version number. Return the 3rd field from the data. i.e saved path with - * #update api - * - * @return available inprogress node path. returns null if not available. - * @throws IOException - */ - String read() throws IOException { - Stat stat = new Stat(); - byte[] data = null; - try { - data = zkc.getData(this.currentInprogressNode, false, stat); - } catch (KeeperException e) { - throw new IOException("Exception while reading the data from " - + currentInprogressNode, e); - } catch (InterruptedException e) { - throw new IOException("Interrupted while reading data from " - + currentInprogressNode, e); - } - this.versionNumberForPermission = stat.getVersion(); - if (data != null) { - CurrentInprogressProto.Builder builder = CurrentInprogressProto.newBuilder(); - TextFormat.merge(new String(data, UTF_8), builder); - if (!builder.isInitialized()) { - throw new IOException("Invalid/Incomplete data in znode"); - } - return builder.build().getPath(); - } else { - LOG.debug("No data available in CurrentInprogress"); - } - return null; - } - - /** Clear the CurrentInprogress node data */ - void clear() throws IOException { - try { - zkc.setData(this.currentInprogressNode, null, versionNumberForPermission); - } catch (KeeperException e) { - throw new IOException( - "Exception when setting the data to CurrentInprogress node", e); - } catch (InterruptedException e) { - throw new IOException( - "Interrupted when setting the data to CurrentInprogress node", e); - } - LOG.debug("Cleared the data from CurrentInprogress"); - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java deleted file mode 100644 index 2d1f8b95c7e..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java +++ /dev/null @@ -1,217 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import java.io.IOException; -import java.util.Comparator; - -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.KeeperException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.EditLogLedgerProto; -import com.google.protobuf.TextFormat; -import static com.google.common.base.Charsets.UTF_8; - -/** - * Utility class for storing the metadata associated - * with a single edit log segment, stored in a single ledger - */ -public class EditLogLedgerMetadata { - static final Log LOG = LogFactory.getLog(EditLogLedgerMetadata.class); - - private String zkPath; - private final int dataLayoutVersion; - private final long ledgerId; - private final long firstTxId; - private long lastTxId; - private boolean inprogress; - - public static final Comparator COMPARATOR - = new Comparator() { - public int compare(EditLogLedgerMetadata o1, - EditLogLedgerMetadata o2) { - if (o1.firstTxId < o2.firstTxId) { - return -1; - } else if (o1.firstTxId == o2.firstTxId) { - return 0; - } else { - return 1; - } - } - }; - - EditLogLedgerMetadata(String zkPath, int dataLayoutVersion, - long ledgerId, long firstTxId) { - this.zkPath = zkPath; - this.dataLayoutVersion = dataLayoutVersion; - this.ledgerId = ledgerId; - this.firstTxId = firstTxId; - this.lastTxId = HdfsServerConstants.INVALID_TXID; - this.inprogress = true; - } - - EditLogLedgerMetadata(String zkPath, int dataLayoutVersion, - long ledgerId, long firstTxId, - long lastTxId) { - this.zkPath = zkPath; - this.dataLayoutVersion = dataLayoutVersion; - this.ledgerId = ledgerId; - this.firstTxId = firstTxId; - this.lastTxId = lastTxId; - this.inprogress = false; - } - - String getZkPath() { - return zkPath; - } - - long getFirstTxId() { - return firstTxId; - } - - long getLastTxId() { - return lastTxId; - } - - long getLedgerId() { - return ledgerId; - } - - boolean isInProgress() { - return this.inprogress; - } - - int getDataLayoutVersion() { - return this.dataLayoutVersion; - } - - void finalizeLedger(long newLastTxId) { - assert this.lastTxId == HdfsServerConstants.INVALID_TXID; - this.lastTxId = newLastTxId; - this.inprogress = false; - } - - static EditLogLedgerMetadata read(ZooKeeper zkc, String path) - throws IOException, KeeperException.NoNodeException { - try { - byte[] data = zkc.getData(path, false, null); - - EditLogLedgerProto.Builder builder = EditLogLedgerProto.newBuilder(); - if (LOG.isDebugEnabled()) { - LOG.debug("Reading " + path + " data: " + new String(data, UTF_8)); - } - TextFormat.merge(new String(data, UTF_8), builder); - if (!builder.isInitialized()) { - throw new IOException("Invalid/Incomplete data in znode"); - } - EditLogLedgerProto ledger = builder.build(); - - int dataLayoutVersion = ledger.getDataLayoutVersion(); - long ledgerId = ledger.getLedgerId(); - long firstTxId = ledger.getFirstTxId(); - if (ledger.hasLastTxId()) { - long lastTxId = ledger.getLastTxId(); - return new EditLogLedgerMetadata(path, dataLayoutVersion, - ledgerId, firstTxId, lastTxId); - } else { - return new EditLogLedgerMetadata(path, dataLayoutVersion, - ledgerId, firstTxId); - } - } catch(KeeperException.NoNodeException nne) { - throw nne; - } catch(KeeperException ke) { - throw new IOException("Error reading from zookeeper", ke); - } catch (InterruptedException ie) { - throw new IOException("Interrupted reading from zookeeper", ie); - } - } - - void write(ZooKeeper zkc, String path) - throws IOException, KeeperException.NodeExistsException { - this.zkPath = path; - - EditLogLedgerProto.Builder builder = EditLogLedgerProto.newBuilder(); - builder.setDataLayoutVersion(dataLayoutVersion) - .setLedgerId(ledgerId).setFirstTxId(firstTxId); - - if (!inprogress) { - builder.setLastTxId(lastTxId); - } - try { - zkc.create(path, TextFormat.printToString(builder.build()).getBytes(UTF_8), - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } catch (KeeperException.NodeExistsException nee) { - throw nee; - } catch (KeeperException e) { - throw new IOException("Error creating ledger znode", e); - } catch (InterruptedException ie) { - throw new IOException("Interrupted creating ledger znode", ie); - } - } - - boolean verify(ZooKeeper zkc, String path) { - try { - EditLogLedgerMetadata other = read(zkc, path); - if (LOG.isTraceEnabled()) { - LOG.trace("Verifying " + this.toString() - + " against " + other); - } - return other.equals(this); - } catch (KeeperException e) { - LOG.error("Couldn't verify data in " + path, e); - return false; - } catch (IOException ie) { - LOG.error("Couldn't verify data in " + path, ie); - return false; - } - } - - public boolean equals(Object o) { - if (!(o instanceof EditLogLedgerMetadata)) { - return false; - } - EditLogLedgerMetadata ol = (EditLogLedgerMetadata)o; - return ledgerId == ol.ledgerId - && dataLayoutVersion == ol.dataLayoutVersion - && firstTxId == ol.firstTxId - && lastTxId == ol.lastTxId; - } - - public int hashCode() { - int hash = 1; - hash = hash * 31 + (int) ledgerId; - hash = hash * 31 + (int) firstTxId; - hash = hash * 31 + (int) lastTxId; - hash = hash * 31 + dataLayoutVersion; - return hash; - } - - public String toString() { - return "[LedgerId:"+ledgerId + - ", firstTxId:" + firstTxId + - ", lastTxId:" + lastTxId + - ", dataLayoutVersion:" + dataLayoutVersion + "]"; - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java deleted file mode 100644 index 5a2eefa0a6f..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.data.Stat; - -import org.apache.hadoop.contrib.bkjournal.BKJournalProtos.MaxTxIdProto; -import com.google.protobuf.TextFormat; -import static com.google.common.base.Charsets.UTF_8; - -/** - * Utility class for storing and reading - * the max seen txid in zookeeper - */ -class MaxTxId { - static final Log LOG = LogFactory.getLog(MaxTxId.class); - - private final ZooKeeper zkc; - private final String path; - - private Stat currentStat; - - MaxTxId(ZooKeeper zkc, String path) { - this.zkc = zkc; - this.path = path; - } - - synchronized void store(long maxTxId) throws IOException { - long currentMax = get(); - if (currentMax < maxTxId) { - if (LOG.isTraceEnabled()) { - LOG.trace("Setting maxTxId to " + maxTxId); - } - reset(maxTxId); - } - } - - synchronized void reset(long maxTxId) throws IOException { - try { - MaxTxIdProto.Builder builder = MaxTxIdProto.newBuilder().setTxId(maxTxId); - - byte[] data = TextFormat.printToString(builder.build()).getBytes(UTF_8); - if (currentStat != null) { - currentStat = zkc.setData(path, data, currentStat - .getVersion()); - } else { - zkc.create(path, data, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - } - } catch (KeeperException e) { - throw new IOException("Error writing max tx id", e); - } catch (InterruptedException e) { - throw new IOException("Interrupted while writing max tx id", e); - } - } - - synchronized long get() throws IOException { - try { - currentStat = zkc.exists(path, false); - if (currentStat == null) { - return 0; - } else { - - byte[] bytes = zkc.getData(path, false, currentStat); - - MaxTxIdProto.Builder builder = MaxTxIdProto.newBuilder(); - TextFormat.merge(new String(bytes, UTF_8), builder); - if (!builder.isInitialized()) { - throw new IOException("Invalid/Incomplete data in znode"); - } - - return builder.build().getTxId(); - } - } catch (KeeperException e) { - throw new IOException("Error reading the max tx id from zk", e); - } catch (InterruptedException ie) { - throw new IOException("Interrupted while reading thr max tx id", ie); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/BKJMUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/BKJMUtil.java deleted file mode 100644 index b1fc3d7dbcc..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/BKJMUtil.java +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import static org.junit.Assert.*; - -import java.net.URI; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSConfigKeys; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.KeeperException; - -import org.apache.bookkeeper.proto.BookieServer; -import org.apache.bookkeeper.conf.ServerConfiguration; -import org.apache.bookkeeper.util.LocalBookKeeper; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.List; - -import java.io.IOException; -import java.io.File; - -/** - * Utility class for setting up bookkeeper ensembles - * and bringing individual bookies up and down - */ -class BKJMUtil { - protected static final Log LOG = LogFactory.getLog(BKJMUtil.class); - - int nextPort = 6000; // next port for additionally created bookies - private Thread bkthread = null; - private final static String zkEnsemble = "127.0.0.1:2181"; - int numBookies; - - BKJMUtil(final int numBookies) throws Exception { - this.numBookies = numBookies; - - bkthread = new Thread() { - public void run() { - try { - String[] args = new String[1]; - args[0] = String.valueOf(numBookies); - LOG.info("Starting bk"); - LocalBookKeeper.main(args); - } catch (InterruptedException e) { - // go away quietly - } catch (Exception e) { - LOG.error("Error starting local bk", e); - } - } - }; - } - - void start() throws Exception { - bkthread.start(); - if (!LocalBookKeeper.waitForServerUp(zkEnsemble, 10000)) { - throw new Exception("Error starting zookeeper/bookkeeper"); - } - assertEquals("Not all bookies started", - numBookies, checkBookiesUp(numBookies, 10)); - } - - void teardown() throws Exception { - if (bkthread != null) { - bkthread.interrupt(); - bkthread.join(); - } - } - - static ZooKeeper connectZooKeeper() - throws IOException, KeeperException, InterruptedException { - final CountDownLatch latch = new CountDownLatch(1); - - ZooKeeper zkc = new ZooKeeper(zkEnsemble, 3600, new Watcher() { - public void process(WatchedEvent event) { - if (event.getState() == Watcher.Event.KeeperState.SyncConnected) { - latch.countDown(); - } - } - }); - if (!latch.await(3, TimeUnit.SECONDS)) { - throw new IOException("Zookeeper took too long to connect"); - } - return zkc; - } - - static URI createJournalURI(String path) throws Exception { - return URI.create("bookkeeper://" + zkEnsemble + path); - } - - static void addJournalManagerDefinition(Configuration conf) { - conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_PLUGIN_PREFIX + ".bookkeeper", - "org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager"); - } - - BookieServer newBookie() throws Exception { - int port = nextPort++; - ServerConfiguration bookieConf = new ServerConfiguration(); - bookieConf.setBookiePort(port); - File tmpdir = File.createTempFile("bookie" + Integer.toString(port) + "_", - "test"); - tmpdir.delete(); - tmpdir.mkdir(); - - bookieConf.setZkServers(zkEnsemble); - bookieConf.setJournalDirName(tmpdir.getPath()); - bookieConf.setLedgerDirNames(new String[] { tmpdir.getPath() }); - - BookieServer b = new BookieServer(bookieConf); - b.start(); - for (int i = 0; i < 10 && !b.isRunning(); i++) { - Thread.sleep(10000); - } - if (!b.isRunning()) { - throw new IOException("Bookie would not start"); - } - return b; - } - - /** - * Check that a number of bookies are available - * @param count number of bookies required - * @param timeout number of seconds to wait for bookies to start - * @throws IOException if bookies are not started by the time the timeout hits - */ - int checkBookiesUp(int count, int timeout) throws Exception { - ZooKeeper zkc = connectZooKeeper(); - try { - int mostRecentSize = 0; - for (int i = 0; i < timeout; i++) { - try { - List children = zkc.getChildren("/ledgers/available", - false); - mostRecentSize = children.size(); - // Skip 'readonly znode' which is used for keeping R-O bookie details - if (children.contains("readonly")) { - mostRecentSize = children.size() - 1; - } - if (LOG.isDebugEnabled()) { - LOG.debug("Found " + mostRecentSize + " bookies up, " - + "waiting for " + count); - if (LOG.isTraceEnabled()) { - for (String child : children) { - LOG.trace(" server: " + child); - } - } - } - if (mostRecentSize == count) { - break; - } - } catch (KeeperException e) { - // ignore - } - Thread.sleep(1000); - } - return mostRecentSize; - } finally { - zkc.close(); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java deleted file mode 100644 index ff8c00df039..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java +++ /dev/null @@ -1,414 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import static org.junit.Assert.*; - -import org.junit.Test; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.AfterClass; - -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ha.ServiceFailedException; -import org.apache.hadoop.ha.HAServiceProtocol.RequestSource; -import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; -import org.apache.hadoop.hdfs.DFSConfigKeys; - -import org.apache.hadoop.hdfs.HAUtil; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.MiniDFSNNTopology; - -import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; -import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; - -import org.apache.hadoop.ipc.RemoteException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; - -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.util.ExitUtil; -import org.apache.hadoop.util.ExitUtil.ExitException; - -import org.apache.bookkeeper.proto.BookieServer; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Collection; - -/** - * Integration test to ensure that the BookKeeper JournalManager - * works for HDFS Namenode HA - */ -@RunWith(Parameterized.class) -public class TestBookKeeperAsHASharedDir { - static final Log LOG = LogFactory.getLog(TestBookKeeperAsHASharedDir.class); - - private static BKJMUtil bkutil; - static int numBookies = 3; - - private static final String TEST_FILE_DATA = "HA BookKeeperJournalManager"; - - @Parameters - public static Collection data() { - Collection params = new ArrayList(); - params.add(new Object[]{ Boolean.FALSE }); - params.add(new Object[]{ Boolean.TRUE }); - return params; - } - - private static boolean useAsyncEditLog; - public TestBookKeeperAsHASharedDir(Boolean async) { - useAsyncEditLog = async; - } - - private static Configuration getConf() { - Configuration conf = new Configuration(); - conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING, - useAsyncEditLog); - return conf; - } - - @BeforeClass - public static void setupBookkeeper() throws Exception { - bkutil = new BKJMUtil(numBookies); - bkutil.start(); - } - - @Before - public void clearExitStatus() { - ExitUtil.resetFirstExitException(); - } - - @AfterClass - public static void teardownBookkeeper() throws Exception { - bkutil.teardown(); - } - - /** - * Test simple HA failover usecase with BK - */ - @Test - public void testFailoverWithBK() throws Exception { - MiniDFSCluster cluster = null; - try { - Configuration conf = getConf(); - conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, - BKJMUtil.createJournalURI("/hotfailover").toString()); - BKJMUtil.addJournalManagerDefinition(conf); - - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(MiniDFSNNTopology.simpleHATopology()) - .numDataNodes(0) - .manageNameDfsSharedDirs(false) - .build(); - NameNode nn1 = cluster.getNameNode(0); - NameNode nn2 = cluster.getNameNode(1); - - cluster.waitActive(); - cluster.transitionToActive(0); - - Path p = new Path("/testBKJMfailover"); - - FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); - - fs.mkdirs(p); - cluster.shutdownNameNode(0); - - cluster.transitionToActive(1); - - assertTrue(fs.exists(p)); - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } - } - - /** - * Test HA failover, where BK, as the shared storage, fails. - * Once it becomes available again, a standby can come up. - * Verify that any write happening after the BK fail is not - * available on the standby. - */ - @Test - public void testFailoverWithFailingBKCluster() throws Exception { - int ensembleSize = numBookies + 1; - BookieServer newBookie = bkutil.newBookie(); - assertEquals("New bookie didn't start", - ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10)); - - BookieServer replacementBookie = null; - - MiniDFSCluster cluster = null; - - try { - Configuration conf = getConf(); - conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, - BKJMUtil.createJournalURI("/hotfailoverWithFail").toString()); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE, - ensembleSize); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, - ensembleSize); - BKJMUtil.addJournalManagerDefinition(conf); - - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(MiniDFSNNTopology.simpleHATopology()) - .numDataNodes(0) - .manageNameDfsSharedDirs(false) - .checkExitOnShutdown(false) - .build(); - NameNode nn1 = cluster.getNameNode(0); - NameNode nn2 = cluster.getNameNode(1); - - cluster.waitActive(); - cluster.transitionToActive(0); - - Path p1 = new Path("/testBKJMFailingBKCluster1"); - Path p2 = new Path("/testBKJMFailingBKCluster2"); - - FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); - - fs.mkdirs(p1); - newBookie.shutdown(); // will take down shared storage - assertEquals("New bookie didn't stop", - numBookies, bkutil.checkBookiesUp(numBookies, 10)); - - try { - fs.mkdirs(p2); - fail("mkdirs should result in the NN exiting"); - } catch (RemoteException re) { - assertTrue(re.getClassName().contains("ExitException")); - } - cluster.shutdownNameNode(0); - - try { - cluster.transitionToActive(1); - fail("Shouldn't have been able to transition with bookies down"); - } catch (ExitException ee) { - assertTrue("Should shutdown due to required journal failure", - ee.getMessage().contains( - "starting log segment 3 failed for required journal")); - } - - replacementBookie = bkutil.newBookie(); - assertEquals("Replacement bookie didn't start", - ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10)); - cluster.transitionToActive(1); // should work fine now - - assertTrue(fs.exists(p1)); - assertFalse(fs.exists(p2)); - } finally { - newBookie.shutdown(); - if (replacementBookie != null) { - replacementBookie.shutdown(); - } - - if (cluster != null) { - cluster.shutdown(); - } - } - } - - /** - * Test that two namenodes can't continue as primary - */ - @Test - public void testMultiplePrimariesStarted() throws Exception { - Path p1 = new Path("/testBKJMMultiplePrimary"); - - MiniDFSCluster cluster = null; - try { - Configuration conf = getConf(); - conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, - BKJMUtil.createJournalURI("/hotfailoverMultiple").toString()); - BKJMUtil.addJournalManagerDefinition(conf); - - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(MiniDFSNNTopology.simpleHATopology()) - .numDataNodes(0) - .manageNameDfsSharedDirs(false) - .checkExitOnShutdown(false) - .build(); - NameNode nn1 = cluster.getNameNode(0); - NameNode nn2 = cluster.getNameNode(1); - cluster.waitActive(); - cluster.transitionToActive(0); - - FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); - fs.mkdirs(p1); - nn1.getRpcServer().rollEditLog(); - cluster.transitionToActive(1); - fs = cluster.getFileSystem(0); // get the older active server. - - try { - System.out.println("DMS: > *************"); - boolean foo = fs.delete(p1, true); - System.out.println("DMS: < ************* "+foo); - fail("Log update on older active should cause it to exit"); - } catch (RemoteException re) { - assertTrue(re.getClassName().contains("ExitException")); - } - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } - } - - /** - * Use NameNode INTIALIZESHAREDEDITS to initialize the shared edits. i.e. copy - * the edits log segments to new bkjm shared edits. - * - * @throws Exception - */ - @Test - public void testInitializeBKSharedEdits() throws Exception { - MiniDFSCluster cluster = null; - try { - Configuration conf = getConf(); - HAUtil.setAllowStandbyReads(conf, true); - - MiniDFSNNTopology topology = MiniDFSNNTopology.simpleHATopology(); - cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology) - .numDataNodes(0).build(); - cluster.waitActive(); - // Shutdown and clear the current filebased shared dir. - cluster.shutdownNameNodes(); - File shareddir = new File(cluster.getSharedEditsDir(0, 1)); - assertTrue("Initial Shared edits dir not fully deleted", - FileUtil.fullyDelete(shareddir)); - - // Check namenodes should not start without shared dir. - assertCanNotStartNamenode(cluster, 0); - assertCanNotStartNamenode(cluster, 1); - - // Configure bkjm as new shared edits dir in both namenodes - Configuration nn1Conf = cluster.getConfiguration(0); - Configuration nn2Conf = cluster.getConfiguration(1); - nn1Conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil - .createJournalURI("/initializeSharedEdits").toString()); - nn2Conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil - .createJournalURI("/initializeSharedEdits").toString()); - BKJMUtil.addJournalManagerDefinition(nn1Conf); - BKJMUtil.addJournalManagerDefinition(nn2Conf); - - // Initialize the BKJM shared edits. - assertFalse(NameNode.initializeSharedEdits(nn1Conf)); - - // NameNode should be able to start and should be in sync with BKJM as - // shared dir - assertCanStartHANameNodes(cluster, conf, "/testBKJMInitialize"); - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } - } - - private void assertCanNotStartNamenode(MiniDFSCluster cluster, int nnIndex) { - try { - cluster.restartNameNode(nnIndex, false); - fail("Should not have been able to start NN" + (nnIndex) - + " without shared dir"); - } catch (IOException ioe) { - LOG.info("Got expected exception", ioe); - GenericTestUtils.assertExceptionContains( - "storage directory does not exist or is not accessible", ioe); - } - } - - private void assertCanStartHANameNodes(MiniDFSCluster cluster, - Configuration conf, String path) throws ServiceFailedException, - IOException, URISyntaxException, InterruptedException { - // Now should be able to start both NNs. Pass "false" here so that we don't - // try to waitActive on all NNs, since the second NN doesn't exist yet. - cluster.restartNameNode(0, false); - cluster.restartNameNode(1, true); - - // Make sure HA is working. - cluster - .getNameNode(0) - .getRpcServer() - .transitionToActive( - new StateChangeRequestInfo(RequestSource.REQUEST_BY_USER)); - FileSystem fs = null; - try { - Path newPath = new Path(path); - fs = HATestUtil.configureFailoverFs(cluster, conf); - assertTrue(fs.mkdirs(newPath)); - HATestUtil.waitForStandbyToCatchUp(cluster.getNameNode(0), - cluster.getNameNode(1)); - assertTrue(NameNodeAdapter.getFileInfo(cluster.getNameNode(1), - newPath.toString(), false).isDir()); - } finally { - if (fs != null) { - fs.close(); - } - } - } - - /** - * NameNode should load the edits correctly if the applicable edits are - * present in the BKJM. - */ - @Test - public void testNameNodeMultipleSwitchesUsingBKJM() throws Exception { - MiniDFSCluster cluster = null; - try { - Configuration conf = getConf(); - conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil - .createJournalURI("/correctEditLogSelection").toString()); - BKJMUtil.addJournalManagerDefinition(conf); - - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0) - .manageNameDfsSharedDirs(false).build(); - NameNode nn1 = cluster.getNameNode(0); - NameNode nn2 = cluster.getNameNode(1); - cluster.waitActive(); - cluster.transitionToActive(0); - nn1.getRpcServer().rollEditLog(); // Roll Edits from current Active. - // Transition to standby current active gracefully. - cluster.transitionToStandby(0); - // Make the other Active and Roll edits multiple times - cluster.transitionToActive(1); - nn2.getRpcServer().rollEditLog(); - nn2.getRpcServer().rollEditLog(); - // Now One more failover. So NN1 should be able to failover successfully. - cluster.transitionToStandby(1); - cluster.transitionToActive(0); - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java deleted file mode 100644 index f3f6ce5674f..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperConfiguration.java +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.URI; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.Random; - -import org.apache.bookkeeper.util.LocalBookKeeper; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZKUtil; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.server.NIOServerCnxnFactory; -import org.apache.zookeeper.server.ZooKeeperServer; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; - -public class TestBookKeeperConfiguration { - private static final Log LOG = LogFactory - .getLog(TestBookKeeperConfiguration.class); - private static final int ZK_SESSION_TIMEOUT = 5000; - private static final String HOSTPORT = "127.0.0.1:2181"; - private static final int CONNECTION_TIMEOUT = 30000; - private static NIOServerCnxnFactory serverFactory; - private static ZooKeeperServer zks; - private static ZooKeeper zkc; - private static int ZooKeeperDefaultPort = 2181; - private static File ZkTmpDir; - private BookKeeperJournalManager bkjm; - private static final String BK_ROOT_PATH = "/ledgers"; - - private static ZooKeeper connectZooKeeper(String ensemble) - throws IOException, KeeperException, InterruptedException { - final CountDownLatch latch = new CountDownLatch(1); - - ZooKeeper zkc = new ZooKeeper(HOSTPORT, ZK_SESSION_TIMEOUT, new Watcher() { - public void process(WatchedEvent event) { - if (event.getState() == Watcher.Event.KeeperState.SyncConnected) { - latch.countDown(); - } - } - }); - if (!latch.await(ZK_SESSION_TIMEOUT, TimeUnit.MILLISECONDS)) { - throw new IOException("Zookeeper took too long to connect"); - } - return zkc; - } - - private NamespaceInfo newNSInfo() { - Random r = new Random(); - return new NamespaceInfo(r.nextInt(), "testCluster", "TestBPID", -1); - } - - @BeforeClass - public static void setupZooKeeper() throws Exception { - // create a ZooKeeper server(dataDir, dataLogDir, port) - LOG.info("Starting ZK server"); - ZkTmpDir = File.createTempFile("zookeeper", "test"); - ZkTmpDir.delete(); - ZkTmpDir.mkdir(); - - try { - zks = new ZooKeeperServer(ZkTmpDir, ZkTmpDir, ZooKeeperDefaultPort); - serverFactory = new NIOServerCnxnFactory(); - serverFactory.configure(new InetSocketAddress(ZooKeeperDefaultPort), 10); - serverFactory.startup(zks); - } catch (Exception e) { - LOG.error("Exception while instantiating ZooKeeper", e); - } - - boolean b = LocalBookKeeper.waitForServerUp(HOSTPORT, CONNECTION_TIMEOUT); - LOG.debug("ZooKeeper server up: " + b); - } - - @Before - public void setup() throws Exception { - zkc = connectZooKeeper(HOSTPORT); - try { - ZKUtil.deleteRecursive(zkc, BK_ROOT_PATH); - } catch (KeeperException.NoNodeException e) { - LOG.debug("Ignoring no node exception on cleanup", e); - } catch (Exception e) { - LOG.error("Exception when deleting bookie root path in zk", e); - } - } - - @After - public void teardown() throws Exception { - if (null != zkc) { - zkc.close(); - } - if (null != bkjm) { - bkjm.close(); - } - } - - @AfterClass - public static void teardownZooKeeper() throws Exception { - if (null != zkc) { - zkc.close(); - } - } - - /** - * Verify the BKJM is creating the bookie available path configured in - * 'dfs.namenode.bookkeeperjournal.zk.availablebookies' - */ - @Test - public void testWithConfiguringBKAvailablePath() throws Exception { - // set Bookie available path in the configuration - String bkAvailablePath - = BookKeeperJournalManager.BKJM_ZK_LEDGERS_AVAILABLE_PATH_DEFAULT; - Configuration conf = new Configuration(); - conf.setStrings(BookKeeperJournalManager.BKJM_ZK_LEDGERS_AVAILABLE_PATH, - bkAvailablePath); - Assert.assertNull(bkAvailablePath + " already exists", zkc.exists( - bkAvailablePath, false)); - NamespaceInfo nsi = newNSInfo(); - bkjm = new BookKeeperJournalManager(conf, - URI.create("bookkeeper://" + HOSTPORT + "/hdfsjournal-WithBKPath"), - nsi); - bkjm.format(nsi); - Assert.assertNotNull("Bookie available path : " + bkAvailablePath - + " doesn't exists", zkc.exists(bkAvailablePath, false)); - } - - /** - * Verify the BKJM is creating the bookie available default path, when there - * is no 'dfs.namenode.bookkeeperjournal.zk.availablebookies' configured - */ - @Test - public void testDefaultBKAvailablePath() throws Exception { - Configuration conf = new Configuration(); - Assert.assertNull(BK_ROOT_PATH + " already exists", zkc.exists( - BK_ROOT_PATH, false)); - NamespaceInfo nsi = newNSInfo(); - bkjm = new BookKeeperJournalManager(conf, - URI.create("bookkeeper://" + HOSTPORT + "/hdfsjournal-DefaultBKPath"), - nsi); - bkjm.format(nsi); - Assert.assertNotNull("Bookie available path : " + BK_ROOT_PATH - + " doesn't exists", zkc.exists(BK_ROOT_PATH, false)); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java deleted file mode 100644 index 52e4568e30f..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperEditLogStreams.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; - -import org.apache.bookkeeper.client.BookKeeper; -import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.conf.ClientConfiguration; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.zookeeper.ZooKeeper; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * Unit test for the bkjm's streams - */ -public class TestBookKeeperEditLogStreams { - static final Log LOG = LogFactory.getLog(TestBookKeeperEditLogStreams.class); - - private static BKJMUtil bkutil; - private final static int numBookies = 3; - - @BeforeClass - public static void setupBookkeeper() throws Exception { - bkutil = new BKJMUtil(numBookies); - bkutil.start(); - } - - @AfterClass - public static void teardownBookkeeper() throws Exception { - bkutil.teardown(); - } - - /** - * Test that bkjm will refuse open a stream on an empty - * ledger. - */ - @Test - public void testEmptyInputStream() throws Exception { - ZooKeeper zk = BKJMUtil.connectZooKeeper(); - - BookKeeper bkc = new BookKeeper(new ClientConfiguration(), zk); - try { - LedgerHandle lh = bkc.createLedger(BookKeeper.DigestType.CRC32, "foobar" - .getBytes()); - lh.close(); - - EditLogLedgerMetadata metadata = new EditLogLedgerMetadata("/foobar", - HdfsServerConstants.NAMENODE_LAYOUT_VERSION, lh.getId(), 0x1234); - try { - new BookKeeperEditLogInputStream(lh, metadata, -1); - fail("Shouldn't get this far, should have thrown"); - } catch (IOException ioe) { - assertTrue(ioe.getMessage().contains("Invalid first bk entry to read")); - } - - metadata = new EditLogLedgerMetadata("/foobar", - HdfsServerConstants.NAMENODE_LAYOUT_VERSION, lh.getId(), 0x1234); - try { - new BookKeeperEditLogInputStream(lh, metadata, 0); - fail("Shouldn't get this far, should have thrown"); - } catch (IOException ioe) { - assertTrue(ioe.getMessage().contains("Invalid first bk entry to read")); - } - } finally { - bkc.close(); - zk.close(); - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java deleted file mode 100644 index b8fc30d0e20..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.MiniDFSNNTopology; -import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; -import org.apache.hadoop.hdfs.server.namenode.ha.TestStandbyCheckpoints; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; - -import java.net.BindException; -import java.util.Random; - -/** - * Runs the same tests as TestStandbyCheckpoints, but - * using a bookkeeper journal manager as the shared directory - */ -public class TestBookKeeperHACheckpoints extends TestStandbyCheckpoints { - //overwrite the nn count - static{ - TestStandbyCheckpoints.NUM_NNS = 2; - } - private static BKJMUtil bkutil = null; - static int numBookies = 3; - static int journalCount = 0; - private final Random random = new Random(); - - private static final Log LOG = LogFactory.getLog(TestStandbyCheckpoints.class); - - @SuppressWarnings("rawtypes") - @Override - @Before - public void setupCluster() throws Exception { - Configuration conf = setupCommonConfig(); - conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, - BKJMUtil.createJournalURI("/checkpointing" + journalCount++) - .toString()); - BKJMUtil.addJournalManagerDefinition(conf); - - int retryCount = 0; - while (true) { - try { - int basePort = 10060 + random.nextInt(100) * 2; - MiniDFSNNTopology topology = new MiniDFSNNTopology() - .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(basePort)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(basePort + 1))); - - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(topology) - .numDataNodes(1) - .manageNameDfsSharedDirs(false) - .build(); - cluster.waitActive(); - - setNNs(); - fs = HATestUtil.configureFailoverFs(cluster, conf); - - cluster.transitionToActive(0); - ++retryCount; - break; - } catch (BindException e) { - LOG.info("Set up MiniDFSCluster failed due to port conflicts, retry " - + retryCount + " times"); - } - } - } - - @BeforeClass - public static void startBK() throws Exception { - journalCount = 0; - bkutil = new BKJMUtil(numBookies); - bkutil.start(); - } - - @AfterClass - public static void shutdownBK() throws Exception { - if (bkutil != null) { - bkutil.teardown(); - } - } - - @Override - public void testCheckpointCancellation() throws Exception { - // Overriden as the implementation in the superclass assumes that writes - // are to a file. This should be fixed at some point - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java deleted file mode 100644 index 07fcd720c19..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java +++ /dev/null @@ -1,984 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import static org.junit.Assert.*; -import static org.mockito.Mockito.spy; -import org.junit.Test; -import org.junit.Before; -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.AfterClass; -import org.mockito.Mockito; - -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Callable; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -import org.apache.hadoop.conf.Configuration; - -import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; -import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogTestUtil; -import org.apache.hadoop.hdfs.server.namenode.JournalManager; -import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; -import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; - -import org.apache.bookkeeper.proto.BookieServer; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.ZooDefs.Ids; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -public class TestBookKeeperJournalManager { - static final Log LOG = LogFactory.getLog(TestBookKeeperJournalManager.class); - - private static final long DEFAULT_SEGMENT_SIZE = 1000; - - protected static Configuration conf = new Configuration(); - private ZooKeeper zkc; - private static BKJMUtil bkutil; - static int numBookies = 3; - private BookieServer newBookie; - - @BeforeClass - public static void setupBookkeeper() throws Exception { - bkutil = new BKJMUtil(numBookies); - bkutil.start(); - } - - @AfterClass - public static void teardownBookkeeper() throws Exception { - bkutil.teardown(); - } - - @Before - public void setup() throws Exception { - zkc = BKJMUtil.connectZooKeeper(); - } - - @After - public void teardown() throws Exception { - zkc.close(); - if (newBookie != null) { - newBookie.shutdown(); - } - } - - private NamespaceInfo newNSInfo() { - Random r = new Random(); - return new NamespaceInfo(r.nextInt(), "testCluster", "TestBPID", -1); - } - - @Test - public void testSimpleWrite() throws Exception { - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi); - bkjm.format(nsi); - - EditLogOutputStream out = bkjm.startLogSegment(1, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - for (long i = 1 ; i <= 100; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(1, 100); - - String zkpath = bkjm.finalizedLedgerZNode(1, 100); - - assertNotNull(zkc.exists(zkpath, false)); - assertNull(zkc.exists(bkjm.inprogressZNode(1), false)); - } - - @Test - public void testNumberOfTransactions() throws Exception { - NamespaceInfo nsi = newNSInfo(); - - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi); - bkjm.format(nsi); - - EditLogOutputStream out = bkjm.startLogSegment(1, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - for (long i = 1 ; i <= 100; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(1, 100); - - long numTrans = bkjm.getNumberOfTransactions(1, true); - assertEquals(100, numTrans); - } - - @Test - public void testNumberOfTransactionsWithGaps() throws Exception { - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-gaps"), nsi); - bkjm.format(nsi); - - long txid = 1; - for (long i = 0; i < 3; i++) { - long start = txid; - EditLogOutputStream out = bkjm.startLogSegment(start, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - for (long j = 1 ; j <= DEFAULT_SEGMENT_SIZE; j++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(txid++); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(start, txid-1); - assertNotNull( - zkc.exists(bkjm.finalizedLedgerZNode(start, txid-1), false)); - } - zkc.delete(bkjm.finalizedLedgerZNode(DEFAULT_SEGMENT_SIZE+1, - DEFAULT_SEGMENT_SIZE*2), -1); - - long numTrans = bkjm.getNumberOfTransactions(1, true); - assertEquals(DEFAULT_SEGMENT_SIZE, numTrans); - - try { - numTrans = bkjm.getNumberOfTransactions(DEFAULT_SEGMENT_SIZE+1, true); - fail("Should have thrown corruption exception by this point"); - } catch (JournalManager.CorruptionException ce) { - // if we get here, everything is going good - } - - numTrans = bkjm.getNumberOfTransactions((DEFAULT_SEGMENT_SIZE*2)+1, true); - assertEquals(DEFAULT_SEGMENT_SIZE, numTrans); - } - - @Test - public void testNumberOfTransactionsWithInprogressAtEnd() throws Exception { - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-inprogressAtEnd"), nsi); - bkjm.format(nsi); - - long txid = 1; - for (long i = 0; i < 3; i++) { - long start = txid; - EditLogOutputStream out = bkjm.startLogSegment(start, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - for (long j = 1 ; j <= DEFAULT_SEGMENT_SIZE; j++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(txid++); - out.write(op); - } - - out.close(); - bkjm.finalizeLogSegment(start, (txid-1)); - assertNotNull( - zkc.exists(bkjm.finalizedLedgerZNode(start, (txid-1)), false)); - } - long start = txid; - EditLogOutputStream out = bkjm.startLogSegment(start, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - for (long j = 1 ; j <= DEFAULT_SEGMENT_SIZE/2; j++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(txid++); - out.write(op); - } - out.setReadyToFlush(); - out.flush(); - out.abort(); - out.close(); - - long numTrans = bkjm.getNumberOfTransactions(1, true); - assertEquals((txid-1), numTrans); - } - - /** - * Create a bkjm namespace, write a journal from txid 1, close stream. - * Try to create a new journal from txid 1. Should throw an exception. - */ - @Test - public void testWriteRestartFrom1() throws Exception { - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-restartFrom1"), nsi); - bkjm.format(nsi); - - long txid = 1; - long start = txid; - EditLogOutputStream out = bkjm.startLogSegment(txid, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - for (long j = 1 ; j <= DEFAULT_SEGMENT_SIZE; j++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(txid++); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(start, (txid-1)); - - txid = 1; - try { - out = bkjm.startLogSegment(txid, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - fail("Shouldn't be able to start another journal from " + txid - + " when one already exists"); - } catch (Exception ioe) { - LOG.info("Caught exception as expected", ioe); - } - - // test border case - txid = DEFAULT_SEGMENT_SIZE; - try { - out = bkjm.startLogSegment(txid, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - fail("Shouldn't be able to start another journal from " + txid - + " when one already exists"); - } catch (IOException ioe) { - LOG.info("Caught exception as expected", ioe); - } - - // open journal continuing from before - txid = DEFAULT_SEGMENT_SIZE + 1; - start = txid; - out = bkjm.startLogSegment(start, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - assertNotNull(out); - - for (long j = 1 ; j <= DEFAULT_SEGMENT_SIZE; j++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(txid++); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(start, (txid-1)); - - // open journal arbitarily far in the future - txid = DEFAULT_SEGMENT_SIZE * 4; - out = bkjm.startLogSegment(txid, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - assertNotNull(out); - } - - @Test - public void testTwoWriters() throws Exception { - long start = 1; - NamespaceInfo nsi = newNSInfo(); - - BookKeeperJournalManager bkjm1 = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi); - bkjm1.format(nsi); - - BookKeeperJournalManager bkjm2 = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi); - - - EditLogOutputStream out1 = bkjm1.startLogSegment(start, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - try { - bkjm2.startLogSegment(start, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - fail("Shouldn't have been able to open the second writer"); - } catch (IOException ioe) { - LOG.info("Caught exception as expected", ioe); - }finally{ - out1.close(); - } - } - - @Test - public void testSimpleRead() throws Exception { - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-simpleread"), - nsi); - bkjm.format(nsi); - - final long numTransactions = 10000; - EditLogOutputStream out = bkjm.startLogSegment(1, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);; - for (long i = 1 ; i <= numTransactions; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(1, numTransactions); - - List in = new ArrayList(); - bkjm.selectInputStreams(in, 1, true); - try { - assertEquals(numTransactions, - FSEditLogTestUtil.countTransactionsInStream(in.get(0))); - } finally { - in.get(0).close(); - } - } - - @Test - public void testSimpleRecovery() throws Exception { - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery"), - nsi); - bkjm.format(nsi); - - EditLogOutputStream out = bkjm.startLogSegment(1, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);; - for (long i = 1 ; i <= 100; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.setReadyToFlush(); - out.flush(); - - out.abort(); - out.close(); - - - assertNull(zkc.exists(bkjm.finalizedLedgerZNode(1, 100), false)); - assertNotNull(zkc.exists(bkjm.inprogressZNode(1), false)); - - bkjm.recoverUnfinalizedSegments(); - - assertNotNull(zkc.exists(bkjm.finalizedLedgerZNode(1, 100), false)); - assertNull(zkc.exists(bkjm.inprogressZNode(1), false)); - } - - /** - * Test that if enough bookies fail to prevent an ensemble, - * writes the bookkeeper will fail. Test that when once again - * an ensemble is available, it can continue to write. - */ - @Test - public void testAllBookieFailure() throws Exception { - // bookie to fail - newBookie = bkutil.newBookie(); - BookieServer replacementBookie = null; - - try { - int ensembleSize = numBookies + 1; - assertEquals("New bookie didn't start", - ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10)); - - // ensure that the journal manager has to use all bookies, - // so that a failure will fail the journal manager - Configuration conf = new Configuration(); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE, - ensembleSize); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, - ensembleSize); - long txid = 1; - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-allbookiefailure"), - nsi); - bkjm.format(nsi); - EditLogOutputStream out = bkjm.startLogSegment(txid, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - - for (long i = 1 ; i <= 3; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(txid++); - out.write(op); - } - out.setReadyToFlush(); - out.flush(); - newBookie.shutdown(); - assertEquals("New bookie didn't die", - numBookies, bkutil.checkBookiesUp(numBookies, 10)); - - try { - for (long i = 1 ; i <= 3; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(txid++); - out.write(op); - } - out.setReadyToFlush(); - out.flush(); - fail("should not get to this stage"); - } catch (IOException ioe) { - LOG.debug("Error writing to bookkeeper", ioe); - assertTrue("Invalid exception message", - ioe.getMessage().contains("Failed to write to bookkeeper")); - } - replacementBookie = bkutil.newBookie(); - - assertEquals("New bookie didn't start", - numBookies+1, bkutil.checkBookiesUp(numBookies+1, 10)); - bkjm.recoverUnfinalizedSegments(); - out = bkjm.startLogSegment(txid, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - for (long i = 1 ; i <= 3; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(txid++); - out.write(op); - } - - out.setReadyToFlush(); - out.flush(); - - } catch (Exception e) { - LOG.error("Exception in test", e); - throw e; - } finally { - if (replacementBookie != null) { - replacementBookie.shutdown(); - } - newBookie.shutdown(); - - if (bkutil.checkBookiesUp(numBookies, 30) != numBookies) { - LOG.warn("Not all bookies from this test shut down, expect errors"); - } - } - } - - /** - * Test that a BookKeeper JM can continue to work across the - * failure of a bookie. This should be handled transparently - * by bookkeeper. - */ - @Test - public void testOneBookieFailure() throws Exception { - newBookie = bkutil.newBookie(); - BookieServer replacementBookie = null; - - try { - int ensembleSize = numBookies + 1; - assertEquals("New bookie didn't start", - ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10)); - - // ensure that the journal manager has to use all bookies, - // so that a failure will fail the journal manager - Configuration conf = new Configuration(); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE, - ensembleSize); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, - ensembleSize); - long txid = 1; - - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"), - nsi); - bkjm.format(nsi); - - EditLogOutputStream out = bkjm.startLogSegment(txid, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - for (long i = 1 ; i <= 3; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(txid++); - out.write(op); - } - out.setReadyToFlush(); - out.flush(); - - replacementBookie = bkutil.newBookie(); - assertEquals("replacement bookie didn't start", - ensembleSize+1, bkutil.checkBookiesUp(ensembleSize+1, 10)); - newBookie.shutdown(); - assertEquals("New bookie didn't die", - ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10)); - - for (long i = 1 ; i <= 3; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(txid++); - out.write(op); - } - out.setReadyToFlush(); - out.flush(); - } catch (Exception e) { - LOG.error("Exception in test", e); - throw e; - } finally { - if (replacementBookie != null) { - replacementBookie.shutdown(); - } - newBookie.shutdown(); - - if (bkutil.checkBookiesUp(numBookies, 30) != numBookies) { - LOG.warn("Not all bookies from this test shut down, expect errors"); - } - } - } - - /** - * If a journal manager has an empty inprogress node, ensure that we throw an - * error, as this should not be possible, and some third party has corrupted - * the zookeeper state - */ - @Test - public void testEmptyInprogressNode() throws Exception { - URI uri = BKJMUtil.createJournalURI("/hdfsjournal-emptyInprogress"); - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, - nsi); - bkjm.format(nsi); - - EditLogOutputStream out = bkjm.startLogSegment(1, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);; - for (long i = 1; i <= 100; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(1, 100); - - out = bkjm.startLogSegment(101, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - out.close(); - bkjm.close(); - String inprogressZNode = bkjm.inprogressZNode(101); - zkc.setData(inprogressZNode, new byte[0], -1); - - bkjm = new BookKeeperJournalManager(conf, uri, nsi); - try { - bkjm.recoverUnfinalizedSegments(); - fail("Should have failed. There should be no way of creating" - + " an empty inprogess znode"); - } catch (IOException e) { - // correct behaviour - assertTrue("Exception different than expected", e.getMessage().contains( - "Invalid/Incomplete data in znode")); - } finally { - bkjm.close(); - } - } - - /** - * If a journal manager has an corrupt inprogress node, ensure that we throw - * an error, as this should not be possible, and some third party has - * corrupted the zookeeper state - */ - @Test - public void testCorruptInprogressNode() throws Exception { - URI uri = BKJMUtil.createJournalURI("/hdfsjournal-corruptInprogress"); - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, - nsi); - bkjm.format(nsi); - - EditLogOutputStream out = bkjm.startLogSegment(1, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);; - for (long i = 1; i <= 100; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(1, 100); - - out = bkjm.startLogSegment(101, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - out.close(); - bkjm.close(); - - String inprogressZNode = bkjm.inprogressZNode(101); - zkc.setData(inprogressZNode, "WholeLottaJunk".getBytes(), -1); - - bkjm = new BookKeeperJournalManager(conf, uri, nsi); - try { - bkjm.recoverUnfinalizedSegments(); - fail("Should have failed. There should be no way of creating" - + " an empty inprogess znode"); - } catch (IOException e) { - // correct behaviour - assertTrue("Exception different than expected", e.getMessage().contains( - "has no field named")); - } finally { - bkjm.close(); - } - } - - /** - * Cases can occur where we create a segment but crash before we even have the - * chance to write the START_SEGMENT op. If this occurs we should warn, but - * load as normal - */ - @Test - public void testEmptyInprogressLedger() throws Exception { - URI uri = BKJMUtil.createJournalURI("/hdfsjournal-emptyInprogressLedger"); - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, - nsi); - bkjm.format(nsi); - - EditLogOutputStream out = bkjm.startLogSegment(1, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);; - for (long i = 1; i <= 100; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(1, 100); - - out = bkjm.startLogSegment(101, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - out.close(); - bkjm.close(); - - bkjm = new BookKeeperJournalManager(conf, uri, nsi); - bkjm.recoverUnfinalizedSegments(); - out = bkjm.startLogSegment(101, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - for (long i = 1; i <= 100; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(101, 200); - - bkjm.close(); - } - - /** - * Test that if we fail between finalizing an inprogress and deleting the - * corresponding inprogress znode. - */ - @Test - public void testRefinalizeAlreadyFinalizedInprogress() throws Exception { - URI uri = BKJMUtil - .createJournalURI("/hdfsjournal-refinalizeInprogressLedger"); - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, - nsi); - bkjm.format(nsi); - - EditLogOutputStream out = bkjm.startLogSegment(1, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);; - for (long i = 1; i <= 100; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.close(); - bkjm.close(); - - String inprogressZNode = bkjm.inprogressZNode(1); - String finalizedZNode = bkjm.finalizedLedgerZNode(1, 100); - assertNotNull("inprogress znode doesn't exist", zkc.exists(inprogressZNode, - null)); - assertNull("finalized znode exists", zkc.exists(finalizedZNode, null)); - - byte[] inprogressData = zkc.getData(inprogressZNode, false, null); - - // finalize - bkjm = new BookKeeperJournalManager(conf, uri, nsi); - bkjm.recoverUnfinalizedSegments(); - bkjm.close(); - - assertNull("inprogress znode exists", zkc.exists(inprogressZNode, null)); - assertNotNull("finalized znode doesn't exist", zkc.exists(finalizedZNode, - null)); - - zkc.create(inprogressZNode, inprogressData, Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT); - - // should work fine - bkjm = new BookKeeperJournalManager(conf, uri, nsi); - bkjm.recoverUnfinalizedSegments(); - bkjm.close(); - } - - /** - * Tests that the edit log file meta data reading from ZooKeeper should be - * able to handle the NoNodeException. bkjm.getInputStream(fromTxId, - * inProgressOk) should suppress the NoNodeException and continue. HDFS-3441. - */ - @Test - public void testEditLogFileNotExistsWhenReadingMetadata() throws Exception { - URI uri = BKJMUtil.createJournalURI("/hdfsjournal-editlogfile"); - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri, - nsi); - bkjm.format(nsi); - - try { - // start new inprogress log segment with txid=1 - // and write transactions till txid=50 - String zkpath1 = startAndFinalizeLogSegment(bkjm, 1, 50); - - // start new inprogress log segment with txid=51 - // and write transactions till txid=100 - String zkpath2 = startAndFinalizeLogSegment(bkjm, 51, 100); - - // read the metadata from ZK. Here simulating the situation - // when reading,the edit log metadata can be removed by purger thread. - ZooKeeper zkspy = spy(BKJMUtil.connectZooKeeper()); - bkjm.setZooKeeper(zkspy); - Mockito.doThrow( - new KeeperException.NoNodeException(zkpath2 + " doesn't exists")) - .when(zkspy).getData(zkpath2, false, null); - - List ledgerList = bkjm.getLedgerList(false); - assertEquals("List contains the metadata of non exists path.", 1, - ledgerList.size()); - assertEquals("LogLedgerMetadata contains wrong zk paths.", zkpath1, - ledgerList.get(0).getZkPath()); - } finally { - bkjm.close(); - } - } - - private enum ThreadStatus { - COMPLETED, GOODEXCEPTION, BADEXCEPTION; - }; - - /** - * Tests that concurrent calls to format will still allow one to succeed. - */ - @Test - public void testConcurrentFormat() throws Exception { - final URI uri = BKJMUtil.createJournalURI("/hdfsjournal-concurrentformat"); - final NamespaceInfo nsi = newNSInfo(); - - // populate with data first - BookKeeperJournalManager bkjm - = new BookKeeperJournalManager(conf, uri, nsi); - bkjm.format(nsi); - for (int i = 1; i < 100*2; i += 2) { - bkjm.startLogSegment(i, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - bkjm.finalizeLogSegment(i, i+1); - } - bkjm.close(); - - final int numThreads = 40; - List> threads - = new ArrayList>(); - final CyclicBarrier barrier = new CyclicBarrier(numThreads); - - for (int i = 0; i < numThreads; i++) { - threads.add(new Callable() { - public ThreadStatus call() { - BookKeeperJournalManager bkjm = null; - try { - bkjm = new BookKeeperJournalManager(conf, uri, nsi); - barrier.await(); - bkjm.format(nsi); - return ThreadStatus.COMPLETED; - } catch (IOException ioe) { - LOG.info("Exception formatting ", ioe); - return ThreadStatus.GOODEXCEPTION; - } catch (InterruptedException ie) { - LOG.error("Interrupted. Something is broken", ie); - Thread.currentThread().interrupt(); - return ThreadStatus.BADEXCEPTION; - } catch (Exception e) { - LOG.error("Some other bad exception", e); - return ThreadStatus.BADEXCEPTION; - } finally { - if (bkjm != null) { - try { - bkjm.close(); - } catch (IOException ioe) { - LOG.error("Error closing journal manager", ioe); - } - } - } - } - }); - } - ExecutorService service = Executors.newFixedThreadPool(numThreads); - List> statuses = service.invokeAll(threads, 60, - TimeUnit.SECONDS); - int numCompleted = 0; - for (Future s : statuses) { - assertTrue(s.isDone()); - assertTrue("Thread threw invalid exception", - s.get() == ThreadStatus.COMPLETED - || s.get() == ThreadStatus.GOODEXCEPTION); - if (s.get() == ThreadStatus.COMPLETED) { - numCompleted++; - } - } - LOG.info("Completed " + numCompleted + " formats"); - assertTrue("No thread managed to complete formatting", numCompleted > 0); - } - - @Test(timeout = 120000) - public void testDefaultAckQuorum() throws Exception { - newBookie = bkutil.newBookie(); - int ensembleSize = numBookies + 1; - int quorumSize = numBookies + 1; - // ensure that the journal manager has to use all bookies, - // so that a failure will fail the journal manager - Configuration conf = new Configuration(); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE, - ensembleSize); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, - quorumSize); - // sets 2 secs - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_SEC, - 2); - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"), nsi); - bkjm.format(nsi); - CountDownLatch sleepLatch = new CountDownLatch(1); - sleepBookie(sleepLatch, newBookie); - - EditLogOutputStream out = bkjm.startLogSegment(1, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - int numTransactions = 100; - for (long i = 1; i <= numTransactions; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - try { - out.close(); - bkjm.finalizeLogSegment(1, numTransactions); - - List in = new ArrayList(); - bkjm.selectInputStreams(in, 1, true); - try { - assertEquals(numTransactions, - FSEditLogTestUtil.countTransactionsInStream(in.get(0))); - } finally { - in.get(0).close(); - } - fail("Should throw exception as not enough non-faulty bookies available!"); - } catch (IOException ioe) { - // expected - } - } - - /** - * Test ack quorum feature supported by bookkeeper. Keep ack quorum bookie - * alive and sleep all the other bookies. Now the client would wait for the - * acknowledgement from the ack size bookies and after receiving the success - * response will continue writing. Non ack client will hang long time to add - * entries. - */ - @Test(timeout = 120000) - public void testAckQuorum() throws Exception { - // slow bookie - newBookie = bkutil.newBookie(); - // make quorum size and ensemble size same to avoid the interleave writing - // of the ledger entries - int ensembleSize = numBookies + 1; - int quorumSize = numBookies + 1; - int ackSize = numBookies; - // ensure that the journal manager has to use all bookies, - // so that a failure will fail the journal manager - Configuration conf = new Configuration(); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE, - ensembleSize); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, - quorumSize); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ACK_QUORUM_SIZE, - ackSize); - // sets 60 minutes - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_SEC, - 3600); - - NamespaceInfo nsi = newNSInfo(); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"), nsi); - bkjm.format(nsi); - CountDownLatch sleepLatch = new CountDownLatch(1); - sleepBookie(sleepLatch, newBookie); - - EditLogOutputStream out = bkjm.startLogSegment(1, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - int numTransactions = 100; - for (long i = 1; i <= numTransactions; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(1, numTransactions); - - List in = new ArrayList(); - bkjm.selectInputStreams(in, 1, true); - try { - assertEquals(numTransactions, - FSEditLogTestUtil.countTransactionsInStream(in.get(0))); - } finally { - sleepLatch.countDown(); - in.get(0).close(); - bkjm.close(); - } - } - - /** - * Sleep a bookie until I count down the latch - * - * @param latch - * Latch to wait on - * @param bookie - * bookie server - * @throws Exception - */ - private void sleepBookie(final CountDownLatch l, final BookieServer bookie) - throws Exception { - - Thread sleeper = new Thread() { - public void run() { - try { - bookie.suspendProcessing(); - l.await(60, TimeUnit.SECONDS); - bookie.resumeProcessing(); - } catch (Exception e) { - LOG.error("Error suspending bookie", e); - } - } - }; - sleeper.setName("BookieServerSleeper-" + bookie.getBookie().getId()); - sleeper.start(); - } - - - private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm, - int startTxid, int endTxid) throws IOException, KeeperException, - InterruptedException { - EditLogOutputStream out = bkjm.startLogSegment(startTxid, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - for (long i = startTxid; i <= endTxid; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.close(); - // finalize the inprogress_1 log segment. - bkjm.finalizeLogSegment(startTxid, endTxid); - String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid); - assertNotNull(zkc.exists(zkpath1, false)); - assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false)); - return zkpath1; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperSpeculativeRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperSpeculativeRead.java deleted file mode 100644 index f5b86bc784e..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperSpeculativeRead.java +++ /dev/null @@ -1,167 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import static org.junit.Assert.assertEquals; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.bookkeeper.proto.BookieServer; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; -import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogTestUtil; -import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; -import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import org.apache.zookeeper.ZooKeeper; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestBookKeeperSpeculativeRead { - private static final Log LOG = LogFactory - .getLog(TestBookKeeperSpeculativeRead.class); - - private ZooKeeper zkc; - private static BKJMUtil bkutil; - private static int numLocalBookies = 1; - private static List bks = new ArrayList(); - - @BeforeClass - public static void setupBookkeeper() throws Exception { - bkutil = new BKJMUtil(1); - bkutil.start(); - } - - @AfterClass - public static void teardownBookkeeper() throws Exception { - bkutil.teardown(); - for (BookieServer bk : bks) { - bk.shutdown(); - } - } - - @Before - public void setup() throws Exception { - zkc = BKJMUtil.connectZooKeeper(); - } - - @After - public void teardown() throws Exception { - zkc.close(); - } - - private NamespaceInfo newNSInfo() { - Random r = new Random(); - return new NamespaceInfo(r.nextInt(), "testCluster", "TestBPID", -1); - } - - /** - * Test speculative read feature supported by bookkeeper. Keep one bookie - * alive and sleep all the other bookies. Non spec client will hang for long - * time to read the entries from the bookkeeper. - */ - @Test(timeout = 120000) - public void testSpeculativeRead() throws Exception { - // starting 9 more servers - for (int i = 1; i < 10; i++) { - bks.add(bkutil.newBookie()); - } - NamespaceInfo nsi = newNSInfo(); - Configuration conf = new Configuration(); - int ensembleSize = numLocalBookies + 9; - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE, - ensembleSize); - conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE, - ensembleSize); - conf.setInt( - BookKeeperJournalManager.BKJM_BOOKKEEPER_SPECULATIVE_READ_TIMEOUT_MS, - 100); - // sets 60 minute - conf.setInt( - BookKeeperJournalManager.BKJM_BOOKKEEPER_READ_ENTRY_TIMEOUT_SEC, 3600); - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-specread"), nsi); - bkjm.format(nsi); - - final long numTransactions = 1000; - EditLogOutputStream out = bkjm.startLogSegment(1, - NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - for (long i = 1; i <= numTransactions; i++) { - FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); - op.setTransactionId(i); - out.write(op); - } - out.close(); - bkjm.finalizeLogSegment(1, numTransactions); - - List in = new ArrayList(); - bkjm.selectInputStreams(in, 1, true); - - // sleep 9 bk servers. Now only one server is running and responding to the - // clients - CountDownLatch sleepLatch = new CountDownLatch(1); - for (final BookieServer bookie : bks) { - sleepBookie(sleepLatch, bookie); - } - try { - assertEquals(numTransactions, - FSEditLogTestUtil.countTransactionsInStream(in.get(0))); - } finally { - in.get(0).close(); - sleepLatch.countDown(); - bkjm.close(); - } - } - - /** - * Sleep a bookie until I count down the latch - * - * @param latch - * latch to wait on - * @param bookie - * bookie server - * @throws Exception - */ - private void sleepBookie(final CountDownLatch latch, final BookieServer bookie) - throws Exception { - - Thread sleeper = new Thread() { - public void run() { - try { - bookie.suspendProcessing(); - latch.await(2, TimeUnit.MINUTES); - bookie.resumeProcessing(); - } catch (Exception e) { - LOG.error("Error suspending bookie", e); - } - } - }; - sleeper.setName("BookieServerSleeper-" + bookie.getBookie().getId()); - sleeper.start(); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBootstrapStandbyWithBKJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBootstrapStandbyWithBKJM.java deleted file mode 100644 index ef7f7086dab..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBootstrapStandbyWithBKJM.java +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import java.io.File; -import java.io.FileFilter; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.MiniDFSNNTopology; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; -import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; -import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby; -import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; -import org.apache.hadoop.hdfs.server.namenode.ha.TestStandbyCheckpoints.SlowCodec; -import org.apache.hadoop.io.compress.CompressionCodecFactory; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.google.common.collect.ImmutableList; - -public class TestBootstrapStandbyWithBKJM { - private static BKJMUtil bkutil; - protected MiniDFSCluster cluster; - - @BeforeClass - public static void setupBookkeeper() throws Exception { - bkutil = new BKJMUtil(3); - bkutil.start(); - } - - @AfterClass - public static void teardownBookkeeper() throws Exception { - bkutil.teardown(); - } - - @After - public void teardown() { - if (cluster != null) { - cluster.shutdown(); - cluster = null; - } - } - - @Before - public void setUp() throws Exception { - Configuration conf = new Configuration(); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 5); - conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); - conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil - .createJournalURI("/bootstrapStandby").toString()); - BKJMUtil.addJournalManagerDefinition(conf); - conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, true); - conf.set(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY, - SlowCodec.class.getCanonicalName()); - CompressionCodecFactory.setCodecClasses(conf, - ImmutableList. of(SlowCodec.class)); - MiniDFSNNTopology topology = new MiniDFSNNTopology() - .addNameservice(new MiniDFSNNTopology.NSConf("ns1").addNN( - new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10001)).addNN( - new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10002))); - cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology) - .numDataNodes(1).manageNameDfsSharedDirs(false).build(); - cluster.waitActive(); - } - - /** - * While boostrapping, in_progress transaction entries should be skipped. - * Bootstrap usage for BKJM : "-force", "-nonInteractive", "-skipSharedEditsCheck" - */ - @Test - public void testBootstrapStandbyWithActiveNN() throws Exception { - // make nn0 active - cluster.transitionToActive(0); - - // do ops and generate in-progress edit log data - Configuration confNN1 = cluster.getConfiguration(1); - DistributedFileSystem dfs = (DistributedFileSystem) HATestUtil - .configureFailoverFs(cluster, confNN1); - for (int i = 1; i <= 10; i++) { - dfs.mkdirs(new Path("/test" + i)); - } - dfs.close(); - - // shutdown nn1 and delete its edit log files - cluster.shutdownNameNode(1); - deleteEditLogIfExists(confNN1); - cluster.getNameNodeRpc(0).setSafeMode(SafeModeAction.SAFEMODE_ENTER, true); - cluster.getNameNodeRpc(0).saveNamespace(0, 0); - cluster.getNameNodeRpc(0).setSafeMode(SafeModeAction.SAFEMODE_LEAVE, true); - - // check without -skipSharedEditsCheck, Bootstrap should fail for BKJM - // immediately after saveNamespace - int rc = BootstrapStandby.run(new String[] { "-force", "-nonInteractive" }, - confNN1); - Assert.assertEquals("Mismatches return code", 6, rc); - - // check with -skipSharedEditsCheck - rc = BootstrapStandby.run(new String[] { "-force", "-nonInteractive", - "-skipSharedEditsCheck" }, confNN1); - Assert.assertEquals("Mismatches return code", 0, rc); - - // Checkpoint as fast as we can, in a tight loop. - confNN1.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1); - cluster.restartNameNode(1); - cluster.transitionToStandby(1); - - NameNode nn0 = cluster.getNameNode(0); - HATestUtil.waitForStandbyToCatchUp(nn0, cluster.getNameNode(1)); - long expectedCheckpointTxId = NameNodeAdapter.getNamesystem(nn0) - .getFSImage().getMostRecentCheckpointTxId(); - HATestUtil.waitForCheckpoint(cluster, 1, - ImmutableList.of((int) expectedCheckpointTxId)); - - // Should have copied over the namespace - FSImageTestUtil.assertNNHasCheckpoints(cluster, 1, - ImmutableList.of((int) expectedCheckpointTxId)); - FSImageTestUtil.assertNNFilesMatch(cluster); - } - - private void deleteEditLogIfExists(Configuration confNN1) { - String editDirs = confNN1.get(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY); - String[] listEditDirs = StringUtils.split(editDirs, ','); - Assert.assertTrue("Wrong edit directory path!", listEditDirs.length > 0); - - for (String dir : listEditDirs) { - File curDir = new File(dir, "current"); - File[] listFiles = curDir.listFiles(new FileFilter() { - @Override - public boolean accept(File f) { - if (!f.getName().startsWith("edits")) { - return true; - } - return false; - } - }); - if (listFiles != null && listFiles.length > 0) { - for (File file : listFiles) { - Assert.assertTrue("Failed to delete edit files!", file.delete()); - } - } - } - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java deleted file mode 100644 index 169a8a8f691..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestCurrentInprogress.java +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.contrib.bkjournal; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.bookkeeper.util.LocalBookKeeper; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.server.NIOServerCnxnFactory; -import org.apache.zookeeper.server.ZooKeeperServer; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * Tests that read, update, clear api from CurrentInprogress - */ -public class TestCurrentInprogress { - private static final Log LOG = LogFactory.getLog(TestCurrentInprogress.class); - private static final String CURRENT_NODE_PATH = "/test"; - private static final String HOSTPORT = "127.0.0.1:2181"; - private static final int CONNECTION_TIMEOUT = 30000; - private static NIOServerCnxnFactory serverFactory; - private static ZooKeeperServer zks; - private static ZooKeeper zkc; - private static int ZooKeeperDefaultPort = 2181; - private static File zkTmpDir; - - private static ZooKeeper connectZooKeeper(String ensemble) - throws IOException, KeeperException, InterruptedException { - final CountDownLatch latch = new CountDownLatch(1); - - ZooKeeper zkc = new ZooKeeper(HOSTPORT, 3600, new Watcher() { - public void process(WatchedEvent event) { - if (event.getState() == Watcher.Event.KeeperState.SyncConnected) { - latch.countDown(); - } - } - }); - if (!latch.await(10, TimeUnit.SECONDS)) { - throw new IOException("Zookeeper took too long to connect"); - } - return zkc; - } - - @BeforeClass - public static void setupZooKeeper() throws Exception { - LOG.info("Starting ZK server"); - zkTmpDir = File.createTempFile("zookeeper", "test"); - zkTmpDir.delete(); - zkTmpDir.mkdir(); - try { - zks = new ZooKeeperServer(zkTmpDir, zkTmpDir, ZooKeeperDefaultPort); - serverFactory = new NIOServerCnxnFactory(); - serverFactory.configure(new InetSocketAddress(ZooKeeperDefaultPort), 10); - serverFactory.startup(zks); - } catch (Exception e) { - LOG.error("Exception while instantiating ZooKeeper", e); - } - boolean b = LocalBookKeeper.waitForServerUp(HOSTPORT, CONNECTION_TIMEOUT); - LOG.debug("ZooKeeper server up: " + b); - } - - @AfterClass - public static void shutDownServer() { - if (null != zks) { - zks.shutdown(); - } - zkTmpDir.delete(); - } - - @Before - public void setup() throws Exception { - zkc = connectZooKeeper(HOSTPORT); - } - - @After - public void teardown() throws Exception { - if (null != zkc) { - zkc.close(); - } - - } - - /** - * Tests that read should be able to read the data which updated with update - * api - */ - @Test - public void testReadShouldReturnTheZnodePathAfterUpdate() throws Exception { - String data = "inprogressNode"; - CurrentInprogress ci = new CurrentInprogress(zkc, CURRENT_NODE_PATH); - ci.init(); - ci.update(data); - String inprogressNodePath = ci.read(); - assertEquals("Not returning inprogressZnode", "inprogressNode", - inprogressNodePath); - } - - /** - * Tests that read should return null if we clear the updated data in - * CurrentInprogress node - */ - @Test - public void testReadShouldReturnNullAfterClear() throws Exception { - CurrentInprogress ci = new CurrentInprogress(zkc, CURRENT_NODE_PATH); - ci.init(); - ci.update("myInprogressZnode"); - ci.read(); - ci.clear(); - String inprogressNodePath = ci.read(); - assertEquals("Expecting null to be return", null, inprogressNodePath); - } - - /** - * Tests that update should throw IOE, if version number modifies between read - * and update - */ - @Test(expected = IOException.class) - public void testUpdateShouldFailWithIOEIfVersionNumberChangedAfterRead() - throws Exception { - CurrentInprogress ci = new CurrentInprogress(zkc, CURRENT_NODE_PATH); - ci.init(); - ci.update("myInprogressZnode"); - assertEquals("Not returning myInprogressZnode", "myInprogressZnode", ci - .read()); - // Updating data in-between to change the data to change the version number - ci.update("YourInprogressZnode"); - ci.update("myInprogressZnode"); - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties deleted file mode 100644 index 52aac432644..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/resources/log4j.properties +++ /dev/null @@ -1,55 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - -# -# Bookkeeper Journal Logging Configuration -# - -# Format is " (, )+ - -# DEFAULT: console appender only -log4j.rootLogger=DEBUG, CONSOLE - -# Example with rolling log file -#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE - -# Example with rolling log file and tracing -#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE - -# -# Log INFO level and above messages to the console -# -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -log4j.appender.CONSOLE.Threshold=INFO -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n - -# -# Add ROLLINGFILE to rootLogger to get log file output -# Log DEBUG level and above messages to a log file -log4j.appender.ROLLINGFILE=org.apache.log4j.DailyRollingFileAppender -log4j.appender.ROLLINGFILE.Threshold=DEBUG -log4j.appender.ROLLINGFILE.File=hdfs-namenode.log -log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout -log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n - -# Max log file size of 10MB -log4j.appender.ROLLINGFILE.MaxFileSize=10MB diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index df45e2a6346..10c0ad694ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -70,6 +70,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.webhdfs.ugi.expire.after.access"; public static final int DFS_WEBHDFS_UGI_EXPIRE_AFTER_ACCESS_DEFAULT = 10*60*1000; //10 minutes + public static final String DFS_WEBHDFS_USE_IPC_CALLQ = + "dfs.webhdfs.use.ipc.callq"; + public static final boolean DFS_WEBHDFS_USE_IPC_CALLQ_DEFAULT = true; // HA related configuration public static final String DFS_DATANODE_RESTART_REPLICA_EXPIRY_KEY = "dfs.datanode.restart.replica.expiration"; @@ -992,6 +995,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.disk.balancer.plan.threshold.percent"; public static final int DFS_DISK_BALANCER_PLAN_THRESHOLD_DEFAULT = 10; + public static final String HTTPFS_BUFFER_SIZE_KEY = + "httpfs.buffer.size"; + public static final int HTTP_BUFFER_SIZE_DEFAULT = 4096; // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java index 7b65abfbe02..ea535e9a5f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java @@ -29,6 +29,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KE import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY; +import static org.apache.hadoop.security.SecurityUtil.buildTokenService; import java.io.IOException; import java.net.InetSocketAddress; @@ -56,7 +57,6 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; -import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -281,8 +281,7 @@ public class HAUtil { // exposed to the user via UGI.getCredentials(), otherwise these // cloned tokens may be inadvertently propagated to jobs Token specificToken = - new Token.PrivateToken(haToken); - SecurityUtil.setTokenService(specificToken, singleNNAddr); + haToken.privateClone(buildTokenService(singleNNAddr)); Text alias = new Text( HAUtilClient.buildTokenServicePrefixForLogicalUri( HdfsConstants.HDFS_URI_SCHEME) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 886984a1795..7949439b431 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -103,6 +103,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.util.FoldedTreeSet; import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.server.namenode.CacheManager; import static org.apache.hadoop.hdfs.util.StripedBlockUtil.getInternalBlockLength; @@ -1059,7 +1060,8 @@ public class BlockManager implements BlockStatsMXBean { } // get block locations - final int numCorruptNodes = countNodes(blk).corruptReplicas(); + NumberReplicas numReplicas = countNodes(blk); + final int numCorruptNodes = numReplicas.corruptReplicas(); final int numCorruptReplicas = corruptReplicas.numCorruptReplicas(blk); if (numCorruptNodes != numCorruptReplicas) { LOG.warn("Inconsistent number of corrupt replicas for " @@ -1068,8 +1070,14 @@ public class BlockManager implements BlockStatsMXBean { } final int numNodes = blocksMap.numNodes(blk); - final boolean isCorrupt = numCorruptReplicas != 0 && - numCorruptReplicas == numNodes; + final boolean isCorrupt; + if (blk.isStriped()) { + BlockInfoStriped sblk = (BlockInfoStriped) blk; + isCorrupt = numCorruptReplicas != 0 && + numReplicas.liveReplicas() < sblk.getRealDataBlockNum(); + } else { + isCorrupt = numCorruptReplicas != 0 && numCorruptReplicas == numNodes; + } final int numMachines = isCorrupt ? numNodes: numNodes - numCorruptReplicas; DatanodeStorageInfo[] machines = new DatanodeStorageInfo[numMachines]; final byte[] blockIndices = blk.isStriped() ? new byte[numMachines] : null; @@ -1145,9 +1153,16 @@ public class BlockManager implements BlockStatsMXBean { fileSizeExcludeBlocksUnderConstruction, mode); isComplete = true; } - return new LocatedBlocks(fileSizeExcludeBlocksUnderConstruction, + LocatedBlocks locations = new LocatedBlocks( + fileSizeExcludeBlocksUnderConstruction, isFileUnderConstruction, locatedblocks, lastlb, isComplete, feInfo, ecPolicy); + // Set caching information for the located blocks. + CacheManager cm = namesystem.getCacheManager(); + if (cm != null) { + cm.setCachedLocations(locations); + } + return locations; } } @@ -1766,8 +1781,12 @@ public class BlockManager implements BlockStatsMXBean { private boolean isInNewRack(DatanodeDescriptor[] srcs, DatanodeDescriptor target) { + LOG.debug("check if target {} increases racks, srcs={}", target, + Arrays.asList(srcs)); for (DatanodeDescriptor src : srcs) { - if (src.getNetworkLocation().equals(target.getNetworkLocation())) { + if (!src.isDecommissionInProgress() && + src.getNetworkLocation().equals(target.getNetworkLocation())) { + LOG.debug("the target {} is in the same rack with src {}", target, src); return false; } } @@ -4005,13 +4024,15 @@ public class BlockManager implements BlockStatsMXBean { return; } NumberReplicas repl = countNodes(block); + int pendingNum = pendingReconstruction.getNumReplicas(block); int curExpectedReplicas = getRedundancy(block); - if (isNeededReconstruction(block, repl.liveReplicas())) { - neededReconstruction.update(block, repl.liveReplicas(), + if (!hasEnoughEffectiveReplicas(block, repl, pendingNum, + curExpectedReplicas)) { + neededReconstruction.update(block, repl.liveReplicas() + pendingNum, repl.readOnlyReplicas(), repl.decommissionedAndDecommissioning(), curExpectedReplicas, curReplicasDelta, expectedReplicasDelta); } else { - int oldReplicas = repl.liveReplicas()-curReplicasDelta; + int oldReplicas = repl.liveReplicas() + pendingNum - curReplicasDelta; int oldExpectedReplicas = curExpectedReplicas-expectedReplicasDelta; neededReconstruction.remove(block, oldReplicas, repl.readOnlyReplicas(), repl.decommissionedAndDecommissioning(), oldExpectedReplicas); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index abfa7824ab0..3958c738d9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -49,8 +49,9 @@ import com.google.common.annotations.VisibleForTesting; public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { private static final String enableDebugLogging = - "For more information, please enable DEBUG log level on " - + BlockPlacementPolicy.class.getName(); + "For more information, please enable DEBUG log level on " + + BlockPlacementPolicy.class.getName() + " and " + + NetworkTopology.class.getName(); private static final ThreadLocal debugLoggingBuilder = new ThreadLocal() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java index 6436fabd167..87b36da0b1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java @@ -388,6 +388,10 @@ public class DecommissionManager { * The number of blocks that have been checked on this tick. */ private int numBlocksChecked = 0; + /** + * The number of blocks checked after (re)holding lock. + */ + private int numBlocksCheckedPerLock = 0; /** * The number of nodes that have been checked on this tick. Used for * statistics. @@ -418,6 +422,7 @@ public class DecommissionManager { } // Reset the checked count at beginning of each iteration numBlocksChecked = 0; + numBlocksCheckedPerLock = 0; numNodesChecked = 0; // Check decom progress namesystem.writeLock(); @@ -451,7 +456,8 @@ public class DecommissionManager { iterkey).iterator(); final LinkedList toRemove = new LinkedList<>(); - while (it.hasNext() && !exceededNumBlocksPerCheck()) { + while (it.hasNext() && !exceededNumBlocksPerCheck() && namesystem + .isRunning()) { numNodesChecked++; final Map.Entry> entry = it.next(); @@ -577,7 +583,28 @@ public class DecommissionManager { int decommissionOnlyReplicas = 0; int lowRedundancyInOpenFiles = 0; while (it.hasNext()) { + if (insufficientList == null + && numBlocksCheckedPerLock >= numBlocksPerCheck) { + // During fullscan insufficientlyReplicated will NOT be null, iterator + // will be DN's iterator. So should not yield lock, otherwise + // ConcurrentModificationException could occur. + // Once the fullscan done, iterator will be a copy. So can yield the + // lock. + // Yielding is required in case of block number is greater than the + // configured per-iteration-limit. + namesystem.writeUnlock(); + try { + LOG.debug("Yielded lock during decommission check"); + Thread.sleep(0, 500); + } catch (InterruptedException ignored) { + return; + } + // reset + numBlocksCheckedPerLock = 0; + namesystem.writeLock(); + } numBlocksChecked++; + numBlocksCheckedPerLock++; final BlockInfo block = it.next(); // Remove the block from the list if it's no longer in the block map, // e.g. the containing file has been deleted diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 82c194f9c72..621b02492ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -370,7 +370,7 @@ public interface HdfsServerConstants { String SECURITY_XATTR_UNREADABLE_BY_SUPERUSER = "security.hdfs.unreadable.by.superuser"; String XATTR_ERASURECODING_POLICY = - "raw.hdfs.erasurecoding.policy"; + "system.hdfs.erasurecoding.policy"; long BLOCK_GROUP_INDEX_MASK = 15; byte MAX_BLOCKS_IN_GROUP = 16; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index 9218e9d1693..e55de353878 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -41,6 +41,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.io.nativeio.NativeIOException; import org.apache.hadoop.util.ToolRunner; @@ -269,11 +270,17 @@ public abstract class Storage extends StorageInfo { private String storageUuid = null; // Storage directory identifier. + private final StorageLocation location; public StorageDirectory(File dir) { // default dirType is null this(dir, null, false); } + public StorageDirectory(StorageLocation location) { + // default dirType is null + this(location.getFile(), null, false, location); + } + public StorageDirectory(File dir, StorageDirType dirType) { this(dir, dirType, false); } @@ -294,11 +301,22 @@ public abstract class Storage extends StorageInfo { * disables locking on the storage directory, false enables locking */ public StorageDirectory(File dir, StorageDirType dirType, boolean isShared) { + this(dir, dirType, isShared, null); + } + + public StorageDirectory(File dir, StorageDirType dirType, + boolean isShared, StorageLocation location) { this.root = dir; this.lock = null; this.dirType = dirType; this.isShared = isShared; + this.location = location; + assert location == null || + dir.getAbsolutePath().startsWith( + location.getFile().getAbsolutePath()): + "The storage location and directory should be equal"; } + /** * Get root directory of this storage @@ -861,6 +879,10 @@ public abstract class Storage extends StorageInfo { } return false; } + + public StorageLocation getStorageLocation() { + return location; + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java index fd89611ff20..e3b6da10469 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java @@ -147,10 +147,10 @@ public class BlockPoolSliceStorage extends Storage { * @throws IOException */ private StorageDirectory loadStorageDirectory(NamespaceInfo nsInfo, - File dataDir, StartupOption startOpt, + File dataDir, StorageLocation location, StartupOption startOpt, List> callables, Configuration conf) throws IOException { - StorageDirectory sd = new StorageDirectory(dataDir, null, true); + StorageDirectory sd = new StorageDirectory(dataDir, null, true, location); try { StorageState curState = sd.analyzeStorage(startOpt, this, true); // sd is locked but not opened @@ -208,9 +208,9 @@ public class BlockPoolSliceStorage extends Storage { * @throws IOException on error */ List loadBpStorageDirectories(NamespaceInfo nsInfo, - Collection dataDirs, StartupOption startOpt, - List> callables, Configuration conf) - throws IOException { + Collection dataDirs, StorageLocation location, + StartupOption startOpt, List> callables, + Configuration conf) throws IOException { List succeedDirs = Lists.newArrayList(); try { for (File dataDir : dataDirs) { @@ -220,7 +220,7 @@ public class BlockPoolSliceStorage extends Storage { "attempt to load an used block storage: " + dataDir); } final StorageDirectory sd = loadStorageDirectory( - nsInfo, dataDir, startOpt, callables, conf); + nsInfo, dataDir, location, startOpt, callables, conf); succeedDirs.add(sd); } } catch (IOException e) { @@ -244,12 +244,12 @@ public class BlockPoolSliceStorage extends Storage { * @throws IOException on error */ List recoverTransitionRead(NamespaceInfo nsInfo, - Collection dataDirs, StartupOption startOpt, - List> callables, Configuration conf) - throws IOException { + Collection dataDirs, StorageLocation location, + StartupOption startOpt, List> callables, + Configuration conf) throws IOException { LOG.info("Analyzing storage directories for bpid " + nsInfo.getBlockPoolID()); final List loaded = loadBpStorageDirectories( - nsInfo, dataDirs, startOpt, callables, conf); + nsInfo, dataDirs, location, startOpt, callables, conf); for (StorageDirectory sd : loaded) { addStorageDir(sd); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java index 456dcc1589f..21484fbc31f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java @@ -201,17 +201,17 @@ public class BlockScanner { FsVolumeSpi volume = ref.getVolume(); if (!isEnabled()) { LOG.debug("Not adding volume scanner for {}, because the block " + - "scanner is disabled.", volume.getBasePath()); + "scanner is disabled.", volume); return; } VolumeScanner scanner = scanners.get(volume.getStorageID()); if (scanner != null) { LOG.error("Already have a scanner for volume {}.", - volume.getBasePath()); + volume); return; } LOG.debug("Adding scanner for volume {} (StorageID {})", - volume.getBasePath(), volume.getStorageID()); + volume, volume.getStorageID()); scanner = new VolumeScanner(conf, datanode, ref); scanner.start(); scanners.put(volume.getStorageID(), scanner); @@ -245,7 +245,7 @@ public class BlockScanner { return; } LOG.info("Removing scanner for volume {} (StorageID {})", - volume.getBasePath(), volume.getStorageID()); + volume, volume.getStorageID()); scanner.shutdown(); scanners.remove(volume.getStorageID()); Uninterruptibles.joinUninterruptibly(scanner, 5, TimeUnit.MINUTES); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 6f332b103a8..9d491d68414 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -60,7 +60,6 @@ import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.EOFException; -import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; @@ -81,7 +80,6 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -802,11 +800,7 @@ public class DataNode extends ReconfigurableBase if (locations.isEmpty()) { return; } - Set volumesToRemove = new HashSet<>(); - for (StorageLocation loc : locations) { - volumesToRemove.add(loc.getFile().getAbsoluteFile()); - } - removeVolumes(volumesToRemove, true); + removeVolumes(locations, true); } /** @@ -825,26 +819,22 @@ public class DataNode extends ReconfigurableBase * @throws IOException */ private synchronized void removeVolumes( - final Set absoluteVolumePaths, boolean clearFailure) + final Collection storageLocations, boolean clearFailure) throws IOException { - for (File vol : absoluteVolumePaths) { - Preconditions.checkArgument(vol.isAbsolute()); - } - - if (absoluteVolumePaths.isEmpty()) { + if (storageLocations.isEmpty()) { return; } LOG.info(String.format("Deactivating volumes (clear failure=%b): %s", - clearFailure, Joiner.on(",").join(absoluteVolumePaths))); + clearFailure, Joiner.on(",").join(storageLocations))); IOException ioe = null; // Remove volumes and block infos from FsDataset. - data.removeVolumes(absoluteVolumePaths, clearFailure); + data.removeVolumes(storageLocations, clearFailure); // Remove volumes from DataStorage. try { - storage.removeVolumes(absoluteVolumePaths); + storage.removeVolumes(storageLocations); } catch (IOException e) { ioe = e; } @@ -852,7 +842,7 @@ public class DataNode extends ReconfigurableBase // Set configuration and dataDirs to reflect volume changes. for (Iterator it = dataDirs.iterator(); it.hasNext(); ) { StorageLocation loc = it.next(); - if (absoluteVolumePaths.contains(loc.getFile().getAbsoluteFile())) { + if (storageLocations.contains(loc)) { it.remove(); } } @@ -3292,18 +3282,18 @@ public class DataNode extends ReconfigurableBase * Check the disk error */ private void checkDiskError() { - Set unhealthyDataDirs = data.checkDataDir(); - if (unhealthyDataDirs != null && !unhealthyDataDirs.isEmpty()) { + Set unhealthyLocations = data.checkDataDir(); + if (unhealthyLocations != null && !unhealthyLocations.isEmpty()) { try { // Remove all unhealthy volumes from DataNode. - removeVolumes(unhealthyDataDirs, false); + removeVolumes(unhealthyLocations, false); } catch (IOException e) { LOG.warn("Error occurred when removing unhealthy storage dirs: " + e.getMessage(), e); } StringBuilder sb = new StringBuilder("DataNode failed volumes:"); - for (File dataDir : unhealthyDataDirs) { - sb.append(dataDir.getAbsolutePath() + ";"); + for (StorageLocation location : unhealthyLocations) { + sb.append(location + ";"); } handleDiskError(sb.toString()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index 7e620c2cea3..7c9bea51c81 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -263,9 +263,10 @@ public class DataStorage extends Storage { } private StorageDirectory loadStorageDirectory(DataNode datanode, - NamespaceInfo nsInfo, File dataDir, StartupOption startOpt, - List> callables) throws IOException { - StorageDirectory sd = new StorageDirectory(dataDir, null, false); + NamespaceInfo nsInfo, File dataDir, StorageLocation location, + StartupOption startOpt, List> callables) + throws IOException { + StorageDirectory sd = new StorageDirectory(dataDir, null, false, location); try { StorageState curState = sd.analyzeStorage(startOpt, this, true); // sd is locked but not opened @@ -310,7 +311,7 @@ public class DataStorage extends Storage { * builder later. * * @param datanode DataNode object. - * @param volume the root path of a storage directory. + * @param location the StorageLocation for the storage directory. * @param nsInfos an array of namespace infos. * @return a VolumeBuilder that holds the metadata of this storage directory * and can be added to DataStorage later. @@ -318,8 +319,10 @@ public class DataStorage extends Storage { * * Note that if there is IOException, the state of DataStorage is not modified. */ - public VolumeBuilder prepareVolume(DataNode datanode, File volume, - List nsInfos) throws IOException { + public VolumeBuilder prepareVolume(DataNode datanode, + StorageLocation location, List nsInfos) + throws IOException { + File volume = location.getFile(); if (containsStorageDir(volume)) { final String errorMessage = "Storage directory is in use"; LOG.warn(errorMessage + "."); @@ -327,7 +330,8 @@ public class DataStorage extends Storage { } StorageDirectory sd = loadStorageDirectory( - datanode, nsInfos.get(0), volume, StartupOption.HOTSWAP, null); + datanode, nsInfos.get(0), volume, location, + StartupOption.HOTSWAP, null); VolumeBuilder builder = new VolumeBuilder(this, sd); for (NamespaceInfo nsInfo : nsInfos) { @@ -338,7 +342,8 @@ public class DataStorage extends Storage { final BlockPoolSliceStorage bpStorage = getBlockPoolSliceStorage(nsInfo); final List dirs = bpStorage.loadBpStorageDirectories( - nsInfo, bpDataDirs, StartupOption.HOTSWAP, null, datanode.getConf()); + nsInfo, bpDataDirs, location, StartupOption.HOTSWAP, + null, datanode.getConf()); builder.addBpStorageDirectories(nsInfo.getBlockPoolID(), dirs); } return builder; @@ -407,7 +412,7 @@ public class DataStorage extends Storage { final List> callables = Lists.newArrayList(); final StorageDirectory sd = loadStorageDirectory( - datanode, nsInfo, root, startOpt, callables); + datanode, nsInfo, root, dataDir, startOpt, callables); if (callables.isEmpty()) { addStorageDir(sd); success.add(dataDir); @@ -458,7 +463,8 @@ public class DataStorage extends Storage { final List> callables = Lists.newArrayList(); final List dirs = bpStorage.recoverTransitionRead( - nsInfo, bpDataDirs, startOpt, callables, datanode.getConf()); + nsInfo, bpDataDirs, dataDir, startOpt, + callables, datanode.getConf()); if (callables.isEmpty()) { for(StorageDirectory sd : dirs) { success.add(sd); @@ -498,9 +504,10 @@ public class DataStorage extends Storage { * @param dirsToRemove a set of storage directories to be removed. * @throws IOException if I/O error when unlocking storage directory. */ - synchronized void removeVolumes(final Set dirsToRemove) + synchronized void removeVolumes( + final Collection storageLocations) throws IOException { - if (dirsToRemove.isEmpty()) { + if (storageLocations.isEmpty()) { return; } @@ -508,7 +515,8 @@ public class DataStorage extends Storage { for (Iterator it = this.storageDirs.iterator(); it.hasNext(); ) { StorageDirectory sd = it.next(); - if (dirsToRemove.contains(sd.getRoot())) { + StorageLocation sdLocation = sd.getStorageLocation(); + if (storageLocations.contains(sdLocation)) { // Remove the block pool level storage first. for (Map.Entry entry : this.bpStorageMap.entrySet()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index c50bfafd2d5..58071dc67d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -37,9 +36,6 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -47,10 +43,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.AutoCloseableLock; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; -import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.ScanInfo; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.util.Time; @@ -209,200 +204,6 @@ public class DirectoryScanner implements Runnable { } } - /** - * Tracks the files and other information related to a block on the disk - * Missing file is indicated by setting the corresponding member - * to null. - * - * Because millions of these structures may be created, we try to save - * memory here. So instead of storing full paths, we store path suffixes. - * The block file, if it exists, will have a path like this: - * / - * So we don't need to store the volume path, since we already know what the - * volume is. - * - * The metadata file, if it exists, will have a path like this: - * /_.meta - * So if we have a block file, there isn't any need to store the block path - * again. - * - * The accessor functions take care of these manipulations. - */ - static class ScanInfo implements Comparable { - private final long blockId; - - /** - * The block file path, relative to the volume's base directory. - * If there was no block file found, this may be null. If 'vol' - * is null, then this is the full path of the block file. - */ - private final String blockSuffix; - - /** - * The suffix of the meta file path relative to the block file. - * If blockSuffix is null, then this will be the entire path relative - * to the volume base directory, or an absolute path if vol is also - * null. - */ - private final String metaSuffix; - - private final FsVolumeSpi volume; - - /** - * Get the file's length in async block scan - */ - private final long blockFileLength; - - private final static Pattern CONDENSED_PATH_REGEX = - Pattern.compile("(?>>32)); - } - - public long getGenStamp() { - return metaSuffix != null ? Block.getGenerationStamp( - getMetaFile().getName()) : - HdfsConstants.GRANDFATHER_GENERATION_STAMP; - } - } /** * Create a new directory scanner, but don't cycle it running yet. @@ -644,7 +445,7 @@ public class DirectoryScanner implements Runnable { // There may be multiple on-disk records for the same block, don't increment // the memory record pointer if so. ScanInfo nextInfo = blockpoolReport[Math.min(d, blockpoolReport.length - 1)]; - if (nextInfo.getBlockId() != info.blockId) { + if (nextInfo.getBlockId() != info.getBlockId()) { ++m; } } else { @@ -762,19 +563,6 @@ public class DirectoryScanner implements Runnable { return list.toSortedArrays(); } - /** - * Helper method to determine if a file name is consistent with a block. - * meta-data file - * - * @param blockId the block ID - * @param metaFile the file to check - * @return whether the file name is a block meta-data file name - */ - private static boolean isBlockMetaFile(String blockId, String metaFile) { - return metaFile.startsWith(blockId) - && metaFile.endsWith(Block.METADATA_EXTENSION); - } - /** * The ReportCompiler class encapsulates the process of searching a datanode's * disks for block information. It operates by performing a DFS of the @@ -784,7 +572,7 @@ public class DirectoryScanner implements Runnable { * ScanInfo object for it and adds that object to its report list. The report * list is returned by the {@link #call()} method. */ - private class ReportCompiler implements Callable { + public class ReportCompiler implements Callable { private final FsVolumeSpi volume; private final DataNode datanode; // Variable for tracking time spent running for throttling purposes @@ -816,14 +604,12 @@ public class DirectoryScanner implements Runnable { ScanInfoPerBlockPool result = new ScanInfoPerBlockPool(bpList.length); for (String bpid : bpList) { LinkedList report = new LinkedList<>(); - File bpFinalizedDir = volume.getFinalizedDir(bpid); perfTimer.start(); throttleTimer.start(); try { - result.put(bpid, - compileReport(volume, bpFinalizedDir, bpFinalizedDir, report)); + result.put(bpid, volume.compileReport(bpid, report, this)); } catch (InterruptedException ex) { // Exit quickly and flag the scanner to do the same result = null; @@ -833,107 +619,13 @@ public class DirectoryScanner implements Runnable { return result; } - /** - * Compile a list of {@link ScanInfo} for the blocks in the directory - * given by {@code dir}. - * - * @param vol the volume that contains the directory to scan - * @param bpFinalizedDir the root directory of the directory to scan - * @param dir the directory to scan - * @param report the list onto which blocks reports are placed - */ - private LinkedList compileReport(FsVolumeSpi vol, - File bpFinalizedDir, File dir, LinkedList report) - throws InterruptedException { - - throttle(); - - List fileNames; - try { - fileNames = IOUtils.listDirectory(dir, BlockDirFilter.INSTANCE); - } catch (IOException ioe) { - LOG.warn("Exception occured while compiling report: ", ioe); - // Initiate a check on disk failure. - datanode.checkDiskErrorAsync(); - // Ignore this directory and proceed. - return report; - } - Collections.sort(fileNames); - - /* - * Assumption: In the sorted list of files block file appears immediately - * before block metadata file. This is true for the current naming - * convention for block file blk_ and meta file - * blk__.meta - */ - for (int i = 0; i < fileNames.size(); i++) { - // Make sure this thread can make a timely exit. With a low throttle - // rate, completing a run can take a looooong time. - if (Thread.interrupted()) { - throw new InterruptedException(); - } - - File file = new File(dir, fileNames.get(i)); - if (file.isDirectory()) { - compileReport(vol, bpFinalizedDir, file, report); - continue; - } - if (!Block.isBlockFilename(file)) { - if (isBlockMetaFile(Block.BLOCK_FILE_PREFIX, file.getName())) { - long blockId = Block.getBlockId(file.getName()); - verifyFileLocation(file.getParentFile(), bpFinalizedDir, - blockId); - report.add(new ScanInfo(blockId, null, file, vol)); - } - continue; - } - File blockFile = file; - long blockId = Block.filename2id(file.getName()); - File metaFile = null; - - // Skip all the files that start with block name until - // getting to the metafile for the block - while (i + 1 < fileNames.size()) { - File blkMetaFile = new File(dir, fileNames.get(i + 1)); - if (!(blkMetaFile.isFile() - && blkMetaFile.getName().startsWith(blockFile.getName()))) { - break; - } - i++; - if (isBlockMetaFile(blockFile.getName(), blkMetaFile.getName())) { - metaFile = blkMetaFile; - break; - } - } - verifyFileLocation(blockFile, bpFinalizedDir, blockId); - report.add(new ScanInfo(blockId, blockFile, metaFile, vol)); - } - return report; - } - - /** - * Verify whether the actual directory location of block file has the - * expected directory path computed using its block ID. - */ - private void verifyFileLocation(File actualBlockFile, - File bpFinalizedDir, long blockId) { - File expectedBlockDir = - DatanodeUtil.idToBlockDir(bpFinalizedDir, blockId); - File actualBlockDir = actualBlockFile.getParentFile(); - if (actualBlockDir.compareTo(expectedBlockDir) != 0) { - LOG.warn("Block: " + blockId + - " found in invalid directory. Expected directory: " + - expectedBlockDir + ". Actual directory: " + actualBlockDir); - } - } - /** * Called by the thread before each potential disk scan so that a pause * can be optionally inserted to limit the number of scans per second. * The limit is controlled by * {@link DFSConfigKeys#DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY}. */ - private void throttle() throws InterruptedException { + public void throttle() throws InterruptedException { accumulateTimeRunning(); if ((throttleLimitMsPerSec < 1000) && @@ -963,7 +655,7 @@ public class DirectoryScanner implements Runnable { } } - private enum BlockDirFilter implements FilenameFilter { + public enum BlockDirFilter implements FilenameFilter { INSTANCE; @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java index e7e9105e356..0c75001e5f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java @@ -500,7 +500,8 @@ public class DiskBalancer { references = this.dataset.getFsVolumeReferences(); for (int ndx = 0; ndx < references.size(); ndx++) { FsVolumeSpi vol = references.get(ndx); - storageIDToVolBasePathMap.put(vol.getStorageID(), vol.getBasePath()); + storageIDToVolBasePathMap.put(vol.getStorageID(), + vol.getBaseURI().getPath()); } references.close(); } @@ -1023,7 +1024,7 @@ public class DiskBalancer { openPoolIters(source, poolIters); if (poolIters.size() == 0) { LOG.error("No block pools found on volume. volume : {}. Exiting.", - source.getBasePath()); + source.getBaseURI()); return; } @@ -1033,17 +1034,16 @@ public class DiskBalancer { // Check for the max error count constraint. if (item.getErrorCount() > getMaxError(item)) { LOG.error("Exceeded the max error count. source {}, dest: {} " + - "error count: {}", source.getBasePath(), - dest.getBasePath(), item.getErrorCount()); - this.setExitFlag(); - continue; + "error count: {}", source.getBaseURI(), + dest.getBaseURI(), item.getErrorCount()); + break; } // Check for the block tolerance constraint. if (isCloseEnough(item)) { LOG.info("Copy from {} to {} done. copied {} bytes and {} " + "blocks.", - source.getBasePath(), dest.getBasePath(), + source.getBaseURI(), dest.getBaseURI(), item.getBytesCopied(), item.getBlocksCopied()); this.setExitFlag(); continue; @@ -1053,7 +1053,7 @@ public class DiskBalancer { // we are not able to find any blocks to copy. if (block == null) { LOG.error("No source blocks, exiting the copy. Source: {}, " + - "Dest:{}", source.getBasePath(), dest.getBasePath()); + "Dest:{}", source.getBaseURI(), dest.getBaseURI()); this.setExitFlag(); continue; } @@ -1081,14 +1081,13 @@ public class DiskBalancer { // exiting here. LOG.error("Destination volume: {} does not have enough space to" + " accommodate a block. Block Size: {} Exiting from" + - " copyBlocks.", dest.getBasePath(), block.getNumBytes()); - this.setExitFlag(); - continue; + " copyBlocks.", dest.getBaseURI(), block.getNumBytes()); + break; } LOG.debug("Moved block with size {} from {} to {}", - block.getNumBytes(), source.getBasePath(), - dest.getBasePath()); + block.getNumBytes(), source.getBaseURI(), + dest.getBaseURI()); // Check for the max throughput constraint. // We sleep here to keep the promise that we will not diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplica.java index cbfc9a53575..58febf02cab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplica.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplica.java @@ -39,8 +39,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.Storage; -import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ScanInfo; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.ScanInfo; import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil; import org.apache.hadoop.io.IOUtils; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java index cbbafc37a18..dc63238b80c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java @@ -25,8 +25,8 @@ import java.net.URI; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ScanInfo; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.ScanInfo; import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; import org.apache.hadoop.util.LightWeightResizableGSet; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java index 3162c5c6c9c..75abc1d39a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.util.StringUtils; + /** * Encapsulates the URI and storage medium that together describe a * storage directory. @@ -37,7 +38,7 @@ import org.apache.hadoop.util.StringUtils; * */ @InterfaceAudience.Private -public class StorageLocation { +public class StorageLocation implements Comparable{ final StorageType storageType; final File file; @@ -104,16 +105,37 @@ public class StorageLocation { @Override public boolean equals(Object obj) { - if (obj == this) { - return true; - } else if (obj == null || !(obj instanceof StorageLocation)) { + if (obj == null || !(obj instanceof StorageLocation)) { return false; } - return toString().equals(obj.toString()); + int comp = compareTo((StorageLocation) obj); + return comp == 0; } @Override public int hashCode() { return toString().hashCode(); } + + @Override + public int compareTo(StorageLocation obj) { + if (obj == this) { + return 0; + } else if (obj == null) { + return -1; + } + + StorageLocation otherStorage = (StorageLocation) obj; + if (this.getFile() != null && otherStorage.getFile() != null) { + return this.getFile().getAbsolutePath().compareTo( + otherStorage.getFile().getAbsolutePath()); + } else if (this.getFile() == null && otherStorage.getFile() == null) { + return this.storageType.compareTo(otherStorage.getStorageType()); + } else if (this.getFile() == null) { + return -1; + } else { + return 1; + } + + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java index 3416b53011c..1e44fb66be7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java @@ -217,7 +217,7 @@ public class VolumeScanner extends Thread { public void printStats(StringBuilder p) { p.append(String.format("Block scanner information for volume %s with base" + - " path %s%n", volume.getStorageID(), volume.getBasePath())); + " path %s%n", volume.getStorageID(), volume)); synchronized (stats) { p.append(String.format("Bytes verified in last hour : %57d%n", stats.bytesScannedInPastHour)); @@ -253,20 +253,20 @@ public class VolumeScanner extends Thread { public void setup(VolumeScanner scanner) { LOG.trace("Starting VolumeScanner {}", - scanner.volume.getBasePath()); + scanner.volume); this.scanner = scanner; } public void handle(ExtendedBlock block, IOException e) { FsVolumeSpi volume = scanner.volume; if (e == null) { - LOG.trace("Successfully scanned {} on {}", block, volume.getBasePath()); + LOG.trace("Successfully scanned {} on {}", block, volume); return; } // If the block does not exist anymore, then it's not an error. if (!volume.getDataset().contains(block)) { LOG.debug("Volume {}: block {} is no longer in the dataset.", - volume.getBasePath(), block); + volume, block); return; } // If the block exists, the exception may due to a race with write: @@ -278,11 +278,10 @@ public class VolumeScanner extends Thread { if (e instanceof FileNotFoundException ) { LOG.info("Volume {}: verification failed for {} because of " + "FileNotFoundException. This may be due to a race with write.", - volume.getBasePath(), block); + volume, block); return; } - LOG.warn("Reporting bad " + block + " with volume " - + volume.getBasePath(), e); + LOG.warn("Reporting bad {} on {}", block, volume); try { scanner.datanode.reportBadBlocks(block, volume); } catch (IOException ie) { @@ -305,7 +304,7 @@ public class VolumeScanner extends Thread { handler = new ScanResultHandler(); } this.resultHandler = handler; - setName("VolumeScannerThread(" + volume.getBasePath() + ")"); + setName("VolumeScannerThread(" + volume + ")"); setDaemon(true); } @@ -376,7 +375,7 @@ public class VolumeScanner extends Thread { BlockIterator iter = blockIters.get(idx); if (!iter.atEnd()) { LOG.info("Now scanning bpid {} on volume {}", - iter.getBlockPoolId(), volume.getBasePath()); + iter.getBlockPoolId(), volume); curBlockIter = iter; return 0L; } @@ -385,7 +384,7 @@ public class VolumeScanner extends Thread { if (waitMs <= 0) { iter.rewind(); LOG.info("Now rescanning bpid {} on volume {}, after more than " + - "{} hour(s)", iter.getBlockPoolId(), volume.getBasePath(), + "{} hour(s)", iter.getBlockPoolId(), volume, TimeUnit.HOURS.convert(conf.scanPeriodMs, TimeUnit.MILLISECONDS)); curBlockIter = iter; return 0L; @@ -416,16 +415,16 @@ public class VolumeScanner extends Thread { cblock.getBlockPoolId(), cblock.getBlockId()); if (b == null) { LOG.info("Replica {} was not found in the VolumeMap for volume {}", - cblock, volume.getBasePath()); + cblock, volume); } else { block = new ExtendedBlock(cblock.getBlockPoolId(), b); } } catch (FileNotFoundException e) { LOG.info("FileNotFoundException while finding block {} on volume {}", - cblock, volume.getBasePath()); + cblock, volume); } catch (IOException e) { LOG.warn("I/O error while finding block {} on volume {}", - cblock, volume.getBasePath()); + cblock, volume); } if (block == null) { return -1; // block not found. @@ -642,7 +641,7 @@ public class VolumeScanner extends Thread { @Override public String toString() { - return "VolumeScanner(" + volume.getBasePath() + + return "VolumeScanner(" + volume + ", " + volume.getStorageID() + ")"; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java index 8f976c290c5..a27de9bc2e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java @@ -122,7 +122,7 @@ class StripedBlockReader { "", newConnectedPeer(block, dnAddr, blockToken, source), source, null, stripedReader.getCachingStrategy(), datanode.getTracer(), -1); } catch (IOException e) { - LOG.debug("Exception while creating remote block reader, datanode {}", + LOG.info("Exception while creating remote block reader, datanode {}", source, e); return null; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index b75ed5bea3d..f2ffa833cf8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.channels.ClosedChannelException; import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -206,7 +207,7 @@ public interface FsDatasetSpi extends FSDatasetMBean { * @param clearFailure set true to clear the failure information about the * volumes. */ - void removeVolumes(Set volumes, boolean clearFailure); + void removeVolumes(Collection volumes, boolean clearFailure); /** @return a storage with the given storage ID */ DatanodeStorage getStorage(final String storageUuid); @@ -482,7 +483,7 @@ public interface FsDatasetSpi extends FSDatasetMBean { * Check if all the data directories are healthy * @return A set of unhealthy data directories. */ - Set checkDataDir(); + Set checkDataDir(); /** * Shutdown the FSDataset diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java index 9e161214c86..dbba31d4529 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java @@ -20,10 +20,20 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset; import java.io.Closeable; import java.io.File; import java.io.IOException; +import java.net.URI; import java.nio.channels.ClosedChannelException; +import java.util.LinkedList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.DF; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ReportCompiler; +import org.apache.hadoop.hdfs.server.datanode.StorageLocation; /** * This is an interface for the underlying volume. @@ -48,14 +58,14 @@ public interface FsVolumeSpi { long getAvailable() throws IOException; /** @return the base path to the volume */ - String getBasePath(); + URI getBaseURI(); - /** @return the path to the volume */ - String getPath(String bpid) throws IOException; + DF getUsageStats(Configuration conf); - /** @return the directory for the finalized blocks in the block pool. */ - File getFinalizedDir(String bpid) throws IOException; - + /** @return the {@link StorageLocation} to the volume */ + StorageLocation getStorageLocation(); + + /** @return the {@link StorageType} of the volume */ StorageType getStorageType(); /** Returns true if the volume is NOT backed by persistent storage. */ @@ -186,4 +196,216 @@ public interface FsVolumeSpi { * Get the FSDatasetSpi which this volume is a part of. */ FsDatasetSpi getDataset(); + + /** + * Tracks the files and other information related to a block on the disk + * Missing file is indicated by setting the corresponding member + * to null. + * + * Because millions of these structures may be created, we try to save + * memory here. So instead of storing full paths, we store path suffixes. + * The block file, if it exists, will have a path like this: + * / + * So we don't need to store the volume path, since we already know what the + * volume is. + * + * The metadata file, if it exists, will have a path like this: + * /_.meta + * So if we have a block file, there isn't any need to store the block path + * again. + * + * The accessor functions take care of these manipulations. + */ + public static class ScanInfo implements Comparable { + private final long blockId; + + /** + * The block file path, relative to the volume's base directory. + * If there was no block file found, this may be null. If 'vol' + * is null, then this is the full path of the block file. + */ + private final String blockSuffix; + + /** + * The suffix of the meta file path relative to the block file. + * If blockSuffix is null, then this will be the entire path relative + * to the volume base directory, or an absolute path if vol is also + * null. + */ + private final String metaSuffix; + + private final FsVolumeSpi volume; + + /** + * Get the file's length in async block scan + */ + private final long blockFileLength; + + private final static Pattern CONDENSED_PATH_REGEX = + Pattern.compile("(?>>32)); + } + + public long getGenStamp() { + return metaSuffix != null ? Block.getGenerationStamp( + getMetaFile().getName()) : + HdfsConstants.GRANDFATHER_GENERATION_STAMP; + } + } + + /** + * Compile a list of {@link ScanInfo} for the blocks in + * the block pool with id {@code bpid}. + * + * @param bpid block pool id to scan + * @param report the list onto which blocks reports are placed + * @param reportCompiler + * @throws IOException + */ + LinkedList compileReport(String bpid, + LinkedList report, ReportCompiler reportCompiler) + throws InterruptedException, IOException; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java index c9160cd65a1..b9c731be5ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java @@ -71,8 +71,8 @@ class FsDatasetAsyncDiskService { private final DataNode datanode; private final FsDatasetImpl fsdatasetImpl; private final ThreadGroup threadGroup; - private Map executors - = new HashMap(); + private Map executors + = new HashMap(); private Map> deletedBlockIds = new HashMap>(); private static final int MAX_DELETED_BLOCKS = 64; @@ -91,7 +91,7 @@ class FsDatasetAsyncDiskService { this.threadGroup = new ThreadGroup(getClass().getSimpleName()); } - private void addExecutorForVolume(final File volume) { + private void addExecutorForVolume(final FsVolumeImpl volume) { ThreadFactory threadFactory = new ThreadFactory() { int counter = 0; @@ -115,18 +115,21 @@ class FsDatasetAsyncDiskService { // This can reduce the number of running threads executor.allowCoreThreadTimeOut(true); - executors.put(volume, executor); + executors.put(volume.getStorageID(), executor); } /** * Starts AsyncDiskService for a new volume * @param volume the root of the new data volume. */ - synchronized void addVolume(File volume) { + synchronized void addVolume(FsVolumeImpl volume) { if (executors == null) { throw new RuntimeException("AsyncDiskService is already shutdown"); } - ThreadPoolExecutor executor = executors.get(volume); + if (volume == null) { + throw new RuntimeException("Attempt to add a null volume"); + } + ThreadPoolExecutor executor = executors.get(volume.getStorageID()); if (executor != null) { throw new RuntimeException("Volume " + volume + " is already existed."); } @@ -137,17 +140,17 @@ class FsDatasetAsyncDiskService { * Stops AsyncDiskService for a volume. * @param volume the root of the volume. */ - synchronized void removeVolume(File volume) { + synchronized void removeVolume(String storageId) { if (executors == null) { throw new RuntimeException("AsyncDiskService is already shutdown"); } - ThreadPoolExecutor executor = executors.get(volume); + ThreadPoolExecutor executor = executors.get(storageId); if (executor == null) { - throw new RuntimeException("Can not find volume " + volume - + " to remove."); + throw new RuntimeException("Can not find volume with storageId " + + storageId + " to remove."); } else { executor.shutdown(); - executors.remove(volume); + executors.remove(storageId); } } @@ -162,13 +165,16 @@ class FsDatasetAsyncDiskService { /** * Execute the task sometime in the future, using ThreadPools. */ - synchronized void execute(File root, Runnable task) { + synchronized void execute(FsVolumeImpl volume, Runnable task) { if (executors == null) { throw new RuntimeException("AsyncDiskService is already shutdown"); } - ThreadPoolExecutor executor = executors.get(root); + if (volume == null) { + throw new RuntimeException("A null volume does not have a executor"); + } + ThreadPoolExecutor executor = executors.get(volume.getStorageID()); if (executor == null) { - throw new RuntimeException("Cannot find root " + root + throw new RuntimeException("Cannot find volume " + volume + " for execution of task " + task); } else { executor.execute(task); @@ -185,7 +191,7 @@ class FsDatasetAsyncDiskService { } else { LOG.info("Shutting down all async disk service threads"); - for (Map.Entry e : executors.entrySet()) { + for (Map.Entry e : executors.entrySet()) { e.getValue().shutdown(); } // clear the executor map so that calling execute again will fail. @@ -198,7 +204,7 @@ class FsDatasetAsyncDiskService { public void submitSyncFileRangeRequest(FsVolumeImpl volume, final FileDescriptor fd, final long offset, final long nbytes, final int flags) { - execute(volume.getCurrentDir(), new Runnable() { + execute(volume, new Runnable() { @Override public void run() { try { @@ -220,7 +226,7 @@ class FsDatasetAsyncDiskService { + " replica " + replicaToDelete + " for deletion"); ReplicaFileDeleteTask deletionTask = new ReplicaFileDeleteTask( volumeRef, replicaToDelete, block, trashDirectory); - execute(((FsVolumeImpl) volumeRef.getVolume()).getCurrentDir(), deletionTask); + execute(((FsVolumeImpl) volumeRef.getVolume()), deletionTask); } /** A task for deleting a block file and its associated meta file, as well diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 26a2e9f9ec6..fd747bdf38c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -361,20 +361,22 @@ class FsDatasetImpl implements FsDatasetSpi { */ private static List getInitialVolumeFailureInfos( Collection dataLocations, DataStorage storage) { - Set failedLocationSet = Sets.newHashSetWithExpectedSize( + Set failedLocationSet = Sets.newHashSetWithExpectedSize( dataLocations.size()); for (StorageLocation sl: dataLocations) { - failedLocationSet.add(sl.getFile().getAbsolutePath()); + LOG.info("Adding to failedLocationSet " + sl); + failedLocationSet.add(sl); } for (Iterator it = storage.dirIterator(); it.hasNext(); ) { Storage.StorageDirectory sd = it.next(); - failedLocationSet.remove(sd.getRoot().getAbsolutePath()); + failedLocationSet.remove(sd.getStorageLocation()); + LOG.info("Removing from failedLocationSet " + sd.getStorageLocation()); } List volumeFailureInfos = Lists.newArrayListWithCapacity( failedLocationSet.size()); long failureDate = Time.now(); - for (String failedStorageLocation: failedLocationSet) { + for (StorageLocation failedStorageLocation: failedLocationSet) { volumeFailureInfos.add(new VolumeFailureInfo(failedStorageLocation, failureDate)); } @@ -403,49 +405,55 @@ class FsDatasetImpl implements FsDatasetSpi { new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); - asyncDiskService.addVolume(sd.getCurrentDir()); + asyncDiskService.addVolume((FsVolumeImpl) ref.getVolume()); volumes.addVolume(ref); } } private void addVolume(Collection dataLocations, Storage.StorageDirectory sd) throws IOException { - final File dir = sd.getCurrentDir(); - final StorageType storageType = - getStorageTypeFromLocations(dataLocations, sd.getRoot()); + final StorageLocation storageLocation = sd.getStorageLocation(); // If IOException raises from FsVolumeImpl() or getVolumeMap(), there is // nothing needed to be rolled back to make various data structures, e.g., // storageMap and asyncDiskService, consistent. - FsVolumeImpl fsVolume = new FsVolumeImpl( - this, sd.getStorageUuid(), dir, this.conf, storageType); + FsVolumeImpl fsVolume = new FsVolumeImplBuilder() + .setDataset(this) + .setStorageID(sd.getStorageUuid()) + .setStorageDirectory(sd) + .setConf(this.conf) + .build(); FsVolumeReference ref = fsVolume.obtainReference(); ReplicaMap tempVolumeMap = new ReplicaMap(datasetLock); fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker); - activateVolume(tempVolumeMap, sd, storageType, ref); - LOG.info("Added volume - " + dir + ", StorageType: " + storageType); + activateVolume(tempVolumeMap, sd, storageLocation.getStorageType(), ref); + LOG.info("Added volume - " + storageLocation + ", StorageType: " + + storageLocation.getStorageType()); } @VisibleForTesting - public FsVolumeImpl createFsVolume(String storageUuid, File currentDir, - StorageType storageType) throws IOException { - return new FsVolumeImpl(this, storageUuid, currentDir, conf, storageType); + public FsVolumeImpl createFsVolume(String storageUuid, + Storage.StorageDirectory sd, + final StorageLocation location) throws IOException { + return new FsVolumeImplBuilder() + .setDataset(this) + .setStorageID(storageUuid) + .setStorageDirectory(sd) + .setConf(conf) + .build(); } @Override public void addVolume(final StorageLocation location, final List nsInfos) throws IOException { - final File dir = location.getFile(); - // Prepare volume in DataStorage final DataStorage.VolumeBuilder builder; try { - builder = dataStorage.prepareVolume(datanode, location.getFile(), nsInfos); + builder = dataStorage.prepareVolume(datanode, location, nsInfos); } catch (IOException e) { - volumes.addVolumeFailureInfo(new VolumeFailureInfo( - location.getFile().getAbsolutePath(), Time.now())); + volumes.addVolumeFailureInfo(new VolumeFailureInfo(location, Time.now())); throw e; } @@ -453,7 +461,7 @@ class FsDatasetImpl implements FsDatasetSpi { StorageType storageType = location.getStorageType(); final FsVolumeImpl fsVolume = - createFsVolume(sd.getStorageUuid(), sd.getCurrentDir(), storageType); + createFsVolume(sd.getStorageUuid(), sd, location); final ReplicaMap tempVolumeMap = new ReplicaMap(new AutoCloseableLock()); ArrayList exceptions = Lists.newArrayList(); @@ -482,34 +490,33 @@ class FsDatasetImpl implements FsDatasetSpi { builder.build(); activateVolume(tempVolumeMap, sd, storageType, ref); - LOG.info("Added volume - " + dir + ", StorageType: " + storageType); + LOG.info("Added volume - " + location + ", StorageType: " + storageType); } /** * Removes a set of volumes from FsDataset. - * @param volumesToRemove a set of absolute root path of each volume. + * @param storageLocationsToRemove a set of + * {@link StorageLocation}s for each volume. * @param clearFailure set true to clear failure information. */ @Override - public void removeVolumes(Set volumesToRemove, boolean clearFailure) { - // Make sure that all volumes are absolute path. - for (File vol : volumesToRemove) { - Preconditions.checkArgument(vol.isAbsolute(), - String.format("%s is not absolute path.", vol.getPath())); - } - + public void removeVolumes( + Collection storageLocationsToRemove, + boolean clearFailure) { Map> blkToInvalidate = new HashMap<>(); List storageToRemove = new ArrayList<>(); try (AutoCloseableLock lock = datasetLock.acquire()) { for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) { Storage.StorageDirectory sd = dataStorage.getStorageDir(idx); - final File absRoot = sd.getRoot().getAbsoluteFile(); - if (volumesToRemove.contains(absRoot)) { - LOG.info("Removing " + absRoot + " from FsDataset."); - + final StorageLocation sdLocation = sd.getStorageLocation(); + LOG.info("Checking removing StorageLocation " + + sdLocation + " with id " + sd.getStorageUuid()); + if (storageLocationsToRemove.contains(sdLocation)) { + LOG.info("Removing StorageLocation " + sdLocation + " with id " + + sd.getStorageUuid() + " from FsDataset."); // Disable the volume from the service. - asyncDiskService.removeVolume(sd.getCurrentDir()); - volumes.removeVolume(absRoot, clearFailure); + asyncDiskService.removeVolume(sd.getStorageUuid()); + volumes.removeVolume(sdLocation, clearFailure); volumes.waitVolumeRemoved(5000, datasetLockCondition); // Removed all replica information for the blocks on the volume. @@ -517,12 +524,14 @@ class FsDatasetImpl implements FsDatasetSpi { // not scan disks. for (String bpid : volumeMap.getBlockPoolList()) { List blocks = new ArrayList<>(); - for (Iterator it = volumeMap.replicas(bpid).iterator(); - it.hasNext(); ) { + for (Iterator it = + volumeMap.replicas(bpid).iterator(); it.hasNext();) { ReplicaInfo block = it.next(); - final File absBasePath = - new File(block.getVolume().getBasePath()).getAbsoluteFile(); - if (absBasePath.equals(absRoot)) { + final StorageLocation blockStorageLocation = + block.getVolume().getStorageLocation(); + LOG.info("checking for block " + block.getBlockId() + + " with storageLocation " + blockStorageLocation); + if (blockStorageLocation.equals(sdLocation)) { blocks.add(block); it.remove(); } @@ -625,7 +634,8 @@ class FsDatasetImpl implements FsDatasetSpi { List failedStorageLocations = Lists.newArrayListWithCapacity( infos.length); for (VolumeFailureInfo info: infos) { - failedStorageLocations.add(info.getFailedStorageLocation()); + failedStorageLocations.add( + info.getFailedStorageLocation().getFile().getAbsolutePath()); } return failedStorageLocations.toArray( new String[failedStorageLocations.size()]); @@ -663,7 +673,8 @@ class FsDatasetImpl implements FsDatasetSpi { long lastVolumeFailureDate = 0; long estimatedCapacityLostTotal = 0; for (VolumeFailureInfo info: infos) { - failedStorageLocations.add(info.getFailedStorageLocation()); + failedStorageLocations.add( + info.getFailedStorageLocation().getFile().getAbsolutePath()); long failureDate = info.getFailureDate(); if (failureDate > lastVolumeFailureDate) { lastVolumeFailureDate = failureDate; @@ -960,25 +971,15 @@ class FsDatasetImpl implements FsDatasetSpi { FsVolumeImpl targetVolume = (FsVolumeImpl) volumeRef.getVolume(); // Copy files to temp dir first - File[] blockFiles = copyBlockFiles(block.getBlockId(), - block.getGenerationStamp(), replicaInfo, - targetVolume.getTmpDir(block.getBlockPoolId()), - replicaInfo.isOnTransientStorage(), smallBufferSize, conf); + ReplicaInfo newReplicaInfo = targetVolume.moveBlockToTmpLocation(block, + replicaInfo, smallBufferSize, conf); - ReplicaInfo newReplicaInfo = new ReplicaBuilder(ReplicaState.TEMPORARY) - .setBlockId(replicaInfo.getBlockId()) - .setGenerationStamp(replicaInfo.getGenerationStamp()) - .setFsVolume(targetVolume) - .setDirectoryToUse(blockFiles[0].getParentFile()) - .setBytesToReserve(0) - .build(); - newReplicaInfo.setNumBytes(blockFiles[1].length()); // Finalize the copied files newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo); try (AutoCloseableLock lock = datasetLock.acquire()) { // Increment numBlocks here as this block moved without knowing to BPS FsVolumeImpl volume = (FsVolumeImpl) newReplicaInfo.getVolume(); - volume.getBlockPoolSlice(block.getBlockPoolId()).incrNumBlocks(); + volume.incrNumBlocks(block.getBlockPoolId()); } removeOldReplica(replicaInfo, newReplicaInfo, block.getBlockPoolId()); @@ -2072,7 +2073,7 @@ class FsDatasetImpl implements FsDatasetSpi { * @return the failed volumes. Returns null if no volume failed. */ @Override // FsDatasetSpi - public Set checkDataDir() { + public Set checkDataDir() { return volumes.checkDirs(); } @@ -2250,9 +2251,8 @@ class FsDatasetImpl implements FsDatasetSpi { .setFsVolume(vol) .setDirectoryToUse(diskFile.getParentFile()) .build(); - ((FsVolumeImpl) vol).getBlockPoolSlice(bpid) - .resolveDuplicateReplicas( - memBlockInfo, diskBlockInfo, volumeMap); + ((FsVolumeImpl) vol).resolveDuplicateReplicas(bpid, + memBlockInfo, diskBlockInfo, volumeMap); } } else { if (!diskFile.delete()) { @@ -2803,15 +2803,15 @@ class FsDatasetImpl implements FsDatasetSpi { // Add thread for DISK volume if RamDisk is configured if (ramDiskConfigured && asyncLazyPersistService != null && - !asyncLazyPersistService.queryVolume(v.getCurrentDir())) { - asyncLazyPersistService.addVolume(v.getCurrentDir()); + !asyncLazyPersistService.queryVolume(v)) { + asyncLazyPersistService.addVolume(v); } // Remove thread for DISK volume if RamDisk is not configured if (!ramDiskConfigured && asyncLazyPersistService != null && - asyncLazyPersistService.queryVolume(v.getCurrentDir())) { - asyncLazyPersistService.removeVolume(v.getCurrentDir()); + asyncLazyPersistService.queryVolume(v)) { + asyncLazyPersistService.removeVolume(v); } } @@ -2946,11 +2946,9 @@ class FsDatasetImpl implements FsDatasetSpi { // Move the replica from lazyPersist/ to finalized/ on // the target volume - BlockPoolSlice bpSlice = - replicaState.getLazyPersistVolume().getBlockPoolSlice(bpid); - newReplicaInfo = - bpSlice.activateSavedReplica(replicaInfo, replicaState); + replicaState.getLazyPersistVolume().activateSavedReplica(bpid, + replicaInfo, replicaState); // Update the volumeMap entry. volumeMap.add(bpid, newReplicaInfo); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index 57fab660c71..76af7249ba3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -23,11 +23,13 @@ import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; import java.io.OutputStreamWriter; +import java.net.URI; import java.nio.channels.ClosedChannelException; import java.nio.file.Files; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.Collections; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -56,13 +58,18 @@ import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil; import org.apache.hadoop.hdfs.server.datanode.LocalReplica; import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; import org.apache.hadoop.hdfs.server.datanode.ReplicaBuilder; import org.apache.hadoop.hdfs.server.datanode.LocalReplicaInPipeline; import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipeline; +import org.apache.hadoop.hdfs.server.datanode.StorageLocation; +import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.BlockDirFilter; +import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ReportCompiler; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaTracker.RamDiskReplica; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.CloseableReferenceCount; @@ -102,8 +109,14 @@ public class FsVolumeImpl implements FsVolumeSpi { private final StorageType storageType; private final Map bpSlices = new ConcurrentHashMap(); + + // Refers to the base StorageLocation used to construct this volume + // (i.e., does not include STORAGE_DIR_CURRENT in + // /STORAGE_DIR_CURRENT/) + private final StorageLocation storageLocation; + private final File currentDir; // /current - private final DF usage; + private final DF usage; private final long reserved; private CloseableReferenceCount reference = new CloseableReferenceCount(); @@ -124,19 +137,25 @@ public class FsVolumeImpl implements FsVolumeSpi { */ protected ThreadPoolExecutor cacheExecutor; - FsVolumeImpl(FsDatasetImpl dataset, String storageID, File currentDir, - Configuration conf, StorageType storageType) throws IOException { + FsVolumeImpl(FsDatasetImpl dataset, String storageID, StorageDirectory sd, + Configuration conf) throws IOException { + + if (sd.getStorageLocation() == null) { + throw new IOException("StorageLocation specified for storage directory " + + sd + " is null"); + } this.dataset = dataset; this.storageID = storageID; + this.reservedForReplicas = new AtomicLong(0L); + this.storageLocation = sd.getStorageLocation(); + this.currentDir = sd.getCurrentDir(); + File parent = currentDir.getParentFile(); + this.usage = new DF(parent, conf); + this.storageType = storageLocation.getStorageType(); this.reserved = conf.getLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY + "." + StringUtils.toLowerCase(storageType.toString()), conf.getLong( DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY, DFSConfigKeys.DFS_DATANODE_DU_RESERVED_DEFAULT)); - this.reservedForReplicas = new AtomicLong(0L); - this.currentDir = currentDir; - File parent = currentDir.getParentFile(); - this.usage = new DF(parent, conf); - this.storageType = storageType; this.configuredCapacity = -1; this.conf = conf; cacheExecutor = initializeCacheExecutor(parent); @@ -285,19 +304,20 @@ public class FsVolumeImpl implements FsVolumeSpi { return true; } + @VisibleForTesting File getCurrentDir() { return currentDir; } - File getRbwDir(String bpid) throws IOException { + protected File getRbwDir(String bpid) throws IOException { return getBlockPoolSlice(bpid).getRbwDir(); } - File getLazyPersistDir(String bpid) throws IOException { + protected File getLazyPersistDir(String bpid) throws IOException { return getBlockPoolSlice(bpid).getLazypersistDir(); } - File getTmpDir(String bpid) throws IOException { + protected File getTmpDir(String bpid) throws IOException { return getBlockPoolSlice(bpid).getTmpDir(); } @@ -448,6 +468,7 @@ public class FsVolumeImpl implements FsVolumeSpi { return reserved; } + @VisibleForTesting BlockPoolSlice getBlockPoolSlice(String bpid) throws IOException { BlockPoolSlice bp = bpSlices.get(bpid); if (bp == null) { @@ -457,21 +478,33 @@ public class FsVolumeImpl implements FsVolumeSpi { } @Override - public String getBasePath() { - return currentDir.getParent(); + public URI getBaseURI() { + return new File(currentDir.getParent()).toURI(); } - + + @Override + public DF getUsageStats(Configuration conf) { + if (currentDir != null) { + try { + return new DF(new File(currentDir.getParent()), conf); + } catch (IOException e) { + LOG.error("Unable to get disk statistics for volume " + this); + } + } + return null; + } + + @Override + public StorageLocation getStorageLocation() { + return storageLocation; + } + @Override public boolean isTransientStorage() { return storageType.isTransient(); } - @Override - public String getPath(String bpid) throws IOException { - return getBlockPoolSlice(bpid).getDirectory().getAbsolutePath(); - } - - @Override + @VisibleForTesting public File getFinalizedDir(String bpid) throws IOException { return getBlockPoolSlice(bpid).getFinalizedDir(); } @@ -951,7 +984,7 @@ public class FsVolumeImpl implements FsVolumeSpi { @Override public String toString() { - return currentDir.getAbsolutePath(); + return currentDir != null ? currentDir.getParent() : "NULL"; } void shutdown() { @@ -1189,5 +1222,167 @@ public class FsVolumeImpl implements FsVolumeSpi { dstBlockFile, true, DFSUtilClient.getSmallBufferSize(conf), conf); } + @Override + public LinkedList compileReport(String bpid, + LinkedList report, ReportCompiler reportCompiler) + throws InterruptedException, IOException { + return compileReport(getFinalizedDir(bpid), + getFinalizedDir(bpid), report, reportCompiler); + } + + private LinkedList compileReport(File bpFinalizedDir, + File dir, LinkedList report, ReportCompiler reportCompiler) + throws InterruptedException { + + reportCompiler.throttle(); + + List fileNames; + try { + fileNames = IOUtils.listDirectory(dir, BlockDirFilter.INSTANCE); + } catch (IOException ioe) { + LOG.warn("Exception occured while compiling report: ", ioe); + // Initiate a check on disk failure. + dataset.datanode.checkDiskErrorAsync(); + // Ignore this directory and proceed. + return report; + } + Collections.sort(fileNames); + + /* + * Assumption: In the sorted list of files block file appears immediately + * before block metadata file. This is true for the current naming + * convention for block file blk_ and meta file + * blk__.meta + */ + for (int i = 0; i < fileNames.size(); i++) { + // Make sure this thread can make a timely exit. With a low throttle + // rate, completing a run can take a looooong time. + if (Thread.interrupted()) { + throw new InterruptedException(); + } + + File file = new File(dir, fileNames.get(i)); + if (file.isDirectory()) { + compileReport(bpFinalizedDir, file, report, reportCompiler); + continue; + } + if (!Block.isBlockFilename(file)) { + if (isBlockMetaFile(Block.BLOCK_FILE_PREFIX, file.getName())) { + long blockId = Block.getBlockId(file.getName()); + verifyFileLocation(file.getParentFile(), bpFinalizedDir, + blockId); + report.add(new ScanInfo(blockId, null, file, this)); + } + continue; + } + File blockFile = file; + long blockId = Block.filename2id(file.getName()); + File metaFile = null; + + // Skip all the files that start with block name until + // getting to the metafile for the block + while (i + 1 < fileNames.size()) { + File blkMetaFile = new File(dir, fileNames.get(i + 1)); + if (!(blkMetaFile.isFile() + && blkMetaFile.getName().startsWith(blockFile.getName()))) { + break; + } + i++; + if (isBlockMetaFile(blockFile.getName(), blkMetaFile.getName())) { + metaFile = blkMetaFile; + break; + } + } + verifyFileLocation(blockFile, bpFinalizedDir, blockId); + report.add(new ScanInfo(blockId, blockFile, metaFile, this)); + } + return report; + } + + /** + * Helper method to determine if a file name is consistent with a block. + * meta-data file + * + * @param blockId the block ID + * @param metaFile the file to check + * @return whether the file name is a block meta-data file name + */ + private static boolean isBlockMetaFile(String blockId, String metaFile) { + return metaFile.startsWith(blockId) + && metaFile.endsWith(Block.METADATA_EXTENSION); + } + + /** + * Verify whether the actual directory location of block file has the + * expected directory path computed using its block ID. + */ + private void verifyFileLocation(File actualBlockFile, + File bpFinalizedDir, long blockId) { + File expectedBlockDir = + DatanodeUtil.idToBlockDir(bpFinalizedDir, blockId); + File actualBlockDir = actualBlockFile.getParentFile(); + if (actualBlockDir.compareTo(expectedBlockDir) != 0) { + LOG.warn("Block: " + blockId + + " found in invalid directory. Expected directory: " + + expectedBlockDir + ". Actual directory: " + actualBlockDir); + } + } + + public ReplicaInfo moveBlockToTmpLocation(ExtendedBlock block, + ReplicaInfo replicaInfo, + int smallBufferSize, + Configuration conf) throws IOException { + + File[] blockFiles = FsDatasetImpl.copyBlockFiles(block.getBlockId(), + block.getGenerationStamp(), replicaInfo, + getTmpDir(block.getBlockPoolId()), + replicaInfo.isOnTransientStorage(), smallBufferSize, conf); + + ReplicaInfo newReplicaInfo = new ReplicaBuilder(ReplicaState.TEMPORARY) + .setBlockId(replicaInfo.getBlockId()) + .setGenerationStamp(replicaInfo.getGenerationStamp()) + .setFsVolume(this) + .setDirectoryToUse(blockFiles[0].getParentFile()) + .setBytesToReserve(0) + .build(); + newReplicaInfo.setNumBytes(blockFiles[1].length()); + return newReplicaInfo; + } + + public File[] copyBlockToLazyPersistLocation(String bpId, long blockId, + long genStamp, + ReplicaInfo replicaInfo, + int smallBufferSize, + Configuration conf) throws IOException { + + File lazyPersistDir = getLazyPersistDir(bpId); + if (!lazyPersistDir.exists() && !lazyPersistDir.mkdirs()) { + FsDatasetImpl.LOG.warn("LazyWriter failed to create " + lazyPersistDir); + throw new IOException("LazyWriter fail to find or " + + "create lazy persist dir: " + lazyPersistDir.toString()); + } + + // No FsDatasetImpl lock for the file copy + File[] targetFiles = FsDatasetImpl.copyBlockFiles( + blockId, genStamp, replicaInfo, lazyPersistDir, true, + smallBufferSize, conf); + return targetFiles; + } + + public void incrNumBlocks(String bpid) throws IOException { + getBlockPoolSlice(bpid).incrNumBlocks(); + } + + public void resolveDuplicateReplicas(String bpid, ReplicaInfo memBlockInfo, + ReplicaInfo diskBlockInfo, ReplicaMap volumeMap) throws IOException { + getBlockPoolSlice(bpid).resolveDuplicateReplicas( + memBlockInfo, diskBlockInfo, volumeMap); + } + + public ReplicaInfo activateSavedReplica(String bpid, + ReplicaInfo replicaInfo, RamDiskReplica replicaState) throws IOException { + return getBlockPoolSlice(bpid).activateSavedReplica(replicaInfo, + replicaState); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplBuilder.java new file mode 100644 index 00000000000..a1f7e9183bf --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplBuilder.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; + +/** + * This class is to be used as a builder for {@link FsVolumeImpl} objects. + */ +public class FsVolumeImplBuilder { + + private FsDatasetImpl dataset; + private String storageID; + private StorageDirectory sd; + private Configuration conf; + + public FsVolumeImplBuilder() { + dataset = null; + storageID = null; + sd = null; + conf = null; + } + + FsVolumeImplBuilder setDataset(FsDatasetImpl dataset) { + this.dataset = dataset; + return this; + } + + FsVolumeImplBuilder setStorageID(String id) { + this.storageID = id; + return this; + } + + FsVolumeImplBuilder setStorageDirectory(StorageDirectory sd) { + this.sd = sd; + return this; + } + + FsVolumeImplBuilder setConf(Configuration conf) { + this.conf = conf; + return this; + } + + FsVolumeImpl build() throws IOException { + return new FsVolumeImpl(dataset, storageID, sd, conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index f869008b3cd..cf9c319fdca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import java.io.File; import java.io.IOException; import java.nio.channels.ClosedChannelException; import java.util.ArrayList; @@ -41,6 +40,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy; import org.apache.hadoop.hdfs.server.datanode.BlockScanner; +import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.AutoCloseableLock; @@ -51,8 +51,10 @@ class FsVolumeList { private final CopyOnWriteArrayList volumes = new CopyOnWriteArrayList<>(); // Tracks volume failures, sorted by volume path. - private final Map volumeFailureInfos = - Collections.synchronizedMap(new TreeMap()); + // map from volume storageID to the volume failure info + private final Map volumeFailureInfos = + Collections.synchronizedMap( + new TreeMap()); private final ConcurrentLinkedQueue volumesBeingRemoved = new ConcurrentLinkedQueue<>(); private final AutoCloseableLock checkDirsLock; @@ -234,10 +236,9 @@ class FsVolumeList { * * @return list of all the failed volumes. */ - Set checkDirs() { + Set checkDirs() { try (AutoCloseableLock lock = checkDirsLock.acquire()) { - Set failedVols = null; - + Set failedLocations = null; // Make a copy of volumes for performing modification final List volumeList = getVolumes(); @@ -247,10 +248,10 @@ class FsVolumeList { fsv.checkDirs(); } catch (DiskErrorException e) { FsDatasetImpl.LOG.warn("Removing failed volume " + fsv + ": ", e); - if (failedVols == null) { - failedVols = new HashSet<>(1); + if (failedLocations == null) { + failedLocations = new HashSet<>(1); } - failedVols.add(new File(fsv.getBasePath()).getAbsoluteFile()); + failedLocations.add(fsv.getStorageLocation()); addVolumeFailureInfo(fsv); removeVolume(fsv); } catch (ClosedChannelException e) { @@ -261,13 +262,13 @@ class FsVolumeList { } } - if (failedVols != null && failedVols.size() > 0) { - FsDatasetImpl.LOG.warn("Completed checkDirs. Found " + failedVols.size() - + " failure volumes."); + if (failedLocations != null && failedLocations.size() > 0) { + FsDatasetImpl.LOG.warn("Completed checkDirs. Found " + + failedLocations.size() + " failure volumes."); } waitVolumeRemoved(5000, checkDirsLockCondition); - return failedVols; + return failedLocations; } } @@ -315,7 +316,7 @@ class FsVolumeList { } // If the volume is used to replace a failed volume, it needs to reset the // volume failure info for this volume. - removeVolumeFailureInfo(new File(volume.getBasePath())); + removeVolumeFailureInfo(volume.getStorageLocation()); FsDatasetImpl.LOG.info("Added new volume: " + volume.getStorageID()); } @@ -351,16 +352,15 @@ class FsVolumeList { * @param volume the volume to be removed. * @param clearFailure set true to remove failure info for this volume. */ - void removeVolume(File volume, boolean clearFailure) { + void removeVolume(StorageLocation storageLocation, boolean clearFailure) { for (FsVolumeImpl fsVolume : volumes) { - String basePath = new File(fsVolume.getBasePath()).getAbsolutePath(); - String targetPath = volume.getAbsolutePath(); - if (basePath.equals(targetPath)) { + StorageLocation baseLocation = fsVolume.getStorageLocation(); + if (baseLocation.equals(storageLocation)) { removeVolume(fsVolume); } } if (clearFailure) { - removeVolumeFailureInfo(volume); + removeVolumeFailureInfo(storageLocation); } } @@ -394,13 +394,13 @@ class FsVolumeList { private void addVolumeFailureInfo(FsVolumeImpl vol) { addVolumeFailureInfo(new VolumeFailureInfo( - new File(vol.getBasePath()).getAbsolutePath(), + vol.getStorageLocation(), Time.now(), vol.getCapacity())); } - private void removeVolumeFailureInfo(File vol) { - volumeFailureInfos.remove(vol.getAbsolutePath()); + private void removeVolumeFailureInfo(StorageLocation location) { + volumeFailureInfos.remove(location); } void addBlockPool(final String bpid, final Configuration conf) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java index 9e549f9bb11..d6969c42e6d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java @@ -58,8 +58,8 @@ class RamDiskAsyncLazyPersistService { private final Configuration conf; private final ThreadGroup threadGroup; - private Map executors - = new HashMap(); + private Map executors + = new HashMap(); private final static HdfsConfiguration EMPTY_HDFS_CONF = new HdfsConfiguration(); /** @@ -75,13 +75,14 @@ class RamDiskAsyncLazyPersistService { this.threadGroup = new ThreadGroup(getClass().getSimpleName()); } - private void addExecutorForVolume(final File volume) { + private void addExecutorForVolume(final String storageId) { ThreadFactory threadFactory = new ThreadFactory() { @Override public Thread newThread(Runnable r) { Thread t = new Thread(threadGroup, r); - t.setName("Async RamDisk lazy persist worker for volume " + volume); + t.setName("Async RamDisk lazy persist worker " + + " for volume with id " + storageId); return t; } }; @@ -93,39 +94,41 @@ class RamDiskAsyncLazyPersistService { // This can reduce the number of running threads executor.allowCoreThreadTimeOut(true); - executors.put(volume, executor); + executors.put(storageId, executor); } /** * Starts AsyncLazyPersistService for a new volume * @param volume the root of the new data volume. */ - synchronized void addVolume(File volume) { + synchronized void addVolume(FsVolumeImpl volume) { + String storageId = volume.getStorageID(); if (executors == null) { throw new RuntimeException("AsyncLazyPersistService is already shutdown"); } - ThreadPoolExecutor executor = executors.get(volume); + ThreadPoolExecutor executor = executors.get(storageId); if (executor != null) { throw new RuntimeException("Volume " + volume + " is already existed."); } - addExecutorForVolume(volume); + addExecutorForVolume(storageId); } /** * Stops AsyncLazyPersistService for a volume. * @param volume the root of the volume. */ - synchronized void removeVolume(File volume) { + synchronized void removeVolume(FsVolumeImpl volume) { + String storageId = volume.getStorageID(); if (executors == null) { throw new RuntimeException("AsyncDiskService is already shutdown"); } - ThreadPoolExecutor executor = executors.get(volume); + ThreadPoolExecutor executor = executors.get(storageId); if (executor == null) { - throw new RuntimeException("Can not find volume " + volume - + " to remove."); + throw new RuntimeException("Can not find volume with storage id " + + storageId + " to remove."); } else { executor.shutdown(); - executors.remove(volume); + executors.remove(storageId); } } @@ -135,25 +138,28 @@ class RamDiskAsyncLazyPersistService { * @return true if there is one thread pool for the volume * false otherwise */ - synchronized boolean queryVolume(File volume) { + synchronized boolean queryVolume(FsVolumeImpl volume) { + String storageId = volume.getStorageID(); if (executors == null) { - throw new RuntimeException("AsyncLazyPersistService is already shutdown"); + throw new RuntimeException( + "AsyncLazyPersistService is already shutdown"); } - ThreadPoolExecutor executor = executors.get(volume); + ThreadPoolExecutor executor = executors.get(storageId); return (executor != null); } /** * Execute the task sometime in the future, using ThreadPools. */ - synchronized void execute(File root, Runnable task) { + synchronized void execute(String storageId, Runnable task) { if (executors == null) { - throw new RuntimeException("AsyncLazyPersistService is already shutdown"); + throw new RuntimeException( + "AsyncLazyPersistService is already shutdown"); } - ThreadPoolExecutor executor = executors.get(root); + ThreadPoolExecutor executor = executors.get(storageId); if (executor == null) { - throw new RuntimeException("Cannot find root " + root - + " for execution of task " + task); + throw new RuntimeException("Cannot find root storage volume with id " + + storageId + " for execution of task " + task); } else { executor.execute(task); } @@ -169,7 +175,7 @@ class RamDiskAsyncLazyPersistService { } else { LOG.info("Shutting down all async lazy persist service threads"); - for (Map.Entry e : executors.entrySet()) { + for (Map.Entry e : executors.entrySet()) { e.getValue().shutdown(); } // clear the executor map so that calling execute again will fail. @@ -189,18 +195,11 @@ class RamDiskAsyncLazyPersistService { + bpId + " block id: " + blockId); } - FsVolumeImpl volume = (FsVolumeImpl)target.getVolume(); - File lazyPersistDir = volume.getLazyPersistDir(bpId); - if (!lazyPersistDir.exists() && !lazyPersistDir.mkdirs()) { - FsDatasetImpl.LOG.warn("LazyWriter failed to create " + lazyPersistDir); - throw new IOException("LazyWriter fail to find or create lazy persist dir: " - + lazyPersistDir.toString()); - } - ReplicaLazyPersistTask lazyPersistTask = new ReplicaLazyPersistTask( - bpId, blockId, genStamp, creationTime, replica, - target, lazyPersistDir); - execute(volume.getCurrentDir(), lazyPersistTask); + bpId, blockId, genStamp, creationTime, replica, target); + + FsVolumeImpl volume = (FsVolumeImpl)target.getVolume(); + execute(volume.getStorageID(), lazyPersistTask); } class ReplicaLazyPersistTask implements Runnable { @@ -210,19 +209,17 @@ class RamDiskAsyncLazyPersistService { private final long creationTime; private final ReplicaInfo replicaInfo; private final FsVolumeReference targetVolume; - private final File lazyPersistDir; ReplicaLazyPersistTask(String bpId, long blockId, long genStamp, long creationTime, ReplicaInfo replicaInfo, - FsVolumeReference targetVolume, File lazyPersistDir) { + FsVolumeReference targetVolume) { this.bpId = bpId; this.blockId = blockId; this.genStamp = genStamp; this.creationTime = creationTime; this.replicaInfo = replicaInfo; this.targetVolume = targetVolume; - this.lazyPersistDir = lazyPersistDir; } @Override @@ -241,14 +238,14 @@ class RamDiskAsyncLazyPersistService { final FsDatasetImpl dataset = (FsDatasetImpl)datanode.getFSDataset(); try (FsVolumeReference ref = this.targetVolume) { int smallBufferSize = DFSUtilClient.getSmallBufferSize(EMPTY_HDFS_CONF); - // No FsDatasetImpl lock for the file copy - File targetFiles[] = FsDatasetImpl.copyBlockFiles( - blockId, genStamp, replicaInfo, lazyPersistDir, true, - smallBufferSize, conf); + + FsVolumeImpl volume = (FsVolumeImpl)ref.getVolume(); + File[] targetFiles = volume.copyBlockToLazyPersistLocation(bpId, + blockId, genStamp, replicaInfo, smallBufferSize, conf); // Lock FsDataSetImpl during onCompleteLazyPersist callback dataset.onCompleteLazyPersist(bpId, blockId, - creationTime, targetFiles, (FsVolumeImpl)ref.getVolume()); + creationTime, targetFiles, volume); succeeded = true; } catch (Exception e){ FsDatasetImpl.LOG.warn( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java index c3ce2a4632f..a762785e018 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java @@ -17,11 +17,13 @@ */ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; +import org.apache.hadoop.hdfs.server.datanode.StorageLocation; + /** * Tracks information about failure of a data volume. */ final class VolumeFailureInfo { - private final String failedStorageLocation; + private final StorageLocation failedStorageLocation; private final long failureDate; private final long estimatedCapacityLost; @@ -33,7 +35,8 @@ final class VolumeFailureInfo { * @param failedStorageLocation storage location that has failed * @param failureDate date/time of failure in milliseconds since epoch */ - public VolumeFailureInfo(String failedStorageLocation, long failureDate) { + public VolumeFailureInfo(StorageLocation failedStorageLocation, + long failureDate) { this(failedStorageLocation, failureDate, 0); } @@ -44,8 +47,8 @@ final class VolumeFailureInfo { * @param failureDate date/time of failure in milliseconds since epoch * @param estimatedCapacityLost estimate of capacity lost in bytes */ - public VolumeFailureInfo(String failedStorageLocation, long failureDate, - long estimatedCapacityLost) { + public VolumeFailureInfo(StorageLocation failedStorageLocation, + long failureDate, long estimatedCapacityLost) { this.failedStorageLocation = failedStorageLocation; this.failureDate = failureDate; this.estimatedCapacityLost = estimatedCapacityLost; @@ -56,7 +59,7 @@ final class VolumeFailureInfo { * * @return storage location that has failed */ - public String getFailedStorageLocation() { + public StorageLocation getFailedStorageLocation() { return this.failedStorageLocation; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java index 366dd9b671a..24bf751a968 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java @@ -63,6 +63,7 @@ import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; @@ -902,7 +903,16 @@ public final class CacheManager { return new BatchedListEntries(results, false); } - public void setCachedLocations(LocatedBlock block) { + public void setCachedLocations(LocatedBlocks locations) { + // don't attempt lookups if there are no cached blocks + if (cachedBlocks.size() > 0) { + for (LocatedBlock lb : locations.getLocatedBlocks()) { + setCachedLocations(lb); + } + } + } + + private void setCachedLocations(LocatedBlock block) { CachedBlock cachedBlock = new CachedBlock(block.getBlock().getBlockId(), (short)0, false); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java index 6df9e75d347..4208b53dfa5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java @@ -21,6 +21,10 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; + +import java.util.HashSet; +import java.util.Set; @InterfaceAudience.Private @InterfaceStability.Unstable @@ -35,6 +39,8 @@ public class ContentSummaryComputationContext { private long yieldCount = 0; private long sleepMilliSec = 0; private int sleepNanoSec = 0; + private Set includedNodes = new HashSet<>(); + private Set deletedSnapshottedNodes = new HashSet<>(); /** * Constructor @@ -51,8 +57,8 @@ public class ContentSummaryComputationContext { this.fsn = fsn; this.limitPerRun = limitPerRun; this.nextCountLimit = limitPerRun; - this.counts = new ContentCounts.Builder().build(); - this.snapshotCounts = new ContentCounts.Builder().build(); + setCounts(new ContentCounts.Builder().build()); + setSnapshotCounts(new ContentCounts.Builder().build()); this.sleepMilliSec = sleepMicroSec/1000; this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000); } @@ -82,6 +88,7 @@ public class ContentSummaryComputationContext { } // Have we reached the limit? + ContentCounts counts = getCounts(); long currentCount = counts.getFileCount() + counts.getSymlinkCount() + counts.getDirectoryCount() + @@ -123,14 +130,22 @@ public class ContentSummaryComputationContext { } /** Get the content counts */ - public ContentCounts getCounts() { + public synchronized ContentCounts getCounts() { return counts; } + private synchronized void setCounts(ContentCounts counts) { + this.counts = counts; + } + public ContentCounts getSnapshotCounts() { return snapshotCounts; } + private void setSnapshotCounts(ContentCounts snapshotCounts) { + this.snapshotCounts = snapshotCounts; + } + public BlockStoragePolicySuite getBlockStoragePolicySuite() { Preconditions.checkState((bsps != null || fsn != null), "BlockStoragePolicySuite must be either initialized or available via" + @@ -138,4 +153,77 @@ public class ContentSummaryComputationContext { return (bsps != null) ? bsps: fsn.getBlockManager().getStoragePolicySuite(); } + + /** + * If the node is an INodeReference, resolves it to the actual inode. + * Snapshot diffs represent renamed / moved files as different + * INodeReferences, but the underlying INode it refers to is consistent. + * + * @param node + * @return The referred INode if there is one, else returns the input + * unmodified. + */ + private INode resolveINodeReference(INode node) { + if (node.isReference() && node instanceof INodeReference) { + return ((INodeReference)node).getReferredINode(); + } + return node; + } + + /** + * Reports that a node is about to be included in this summary. Can be used + * either to simply report that a node has been including, or check whether + * a node has already been included. + * + * @param node + * @return true if node has already been included + */ + public boolean nodeIncluded(INode node) { + INode resolvedNode = resolveINodeReference(node); + synchronized (includedNodes) { + if (!includedNodes.contains(resolvedNode)) { + includedNodes.add(resolvedNode); + return false; + } + } + return true; + } + + /** + * Schedules a node that is listed as DELETED in a snapshot's diff to be + * included in the summary at the end of computation. See + * {@link #tallyDeletedSnapshottedINodes()} for more context. + * + * @param node + */ + public void reportDeletedSnapshottedNode(INode node) { + deletedSnapshottedNodes.add(node); + } + + /** + * Finalizes the computation by including all nodes that were reported as + * deleted by a snapshot but have not been already including due to other + * references. + *

    + * Nodes that get renamed are listed in the snapshot's diff as both DELETED + * under the old name and CREATED under the new name. The computation + * relies on nodes to report themselves as being included (via + * {@link #nodeIncluded(INode)} as the only reliable way to determine which + * nodes were renamed within the tree being summarized and which were + * removed (either by deletion or being renamed outside of the tree). + */ + public synchronized void tallyDeletedSnapshottedINodes() { + /* Temporarily create a new counts object so these results can then be + added to both counts and snapshotCounts */ + ContentCounts originalCounts = getCounts(); + setCounts(new ContentCounts.Builder().build()); + for (INode node : deletedSnapshottedNodes) { + if (!nodeIncluded(node)) { + node.computeContentSummary(Snapshot.CURRENT_STATE_ID, this); + } + } + originalCounts.addContents(getCounts()); + snapshotCounts.addContents(getCounts()); + setCounts(originalCounts); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java index 27d8f501d1a..104d8c3dd37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java @@ -34,6 +34,12 @@ public class EncryptionFaultInjector { return instance; } + @VisibleForTesting + public void startFileNoKey() throws IOException {} + + @VisibleForTesting + public void startFileBeforeGenerateKey() throws IOException {} + @VisibleForTesting public void startFileAfterGenerateKey() throws IOException {} } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java index 511c61610e2..ceeccf6ba5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java @@ -260,12 +260,14 @@ public class EncryptionZoneManager { * * @param srcIIP source IIP * @param dstIIP destination IIP - * @param src source path, used for debugging * @throws IOException if the src cannot be renamed to the dst */ - void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src) + void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP) throws IOException { assert dir.hasReadLock(); + if (!hasCreatedEncryptionZone()) { + return; + } final EncryptionZoneInt srcParentEZI = getParentEncryptionZoneForPath(srcIIP); final EncryptionZoneInt dstParentEZI = @@ -274,17 +276,17 @@ public class EncryptionZoneManager { final boolean dstInEZ = (dstParentEZI != null); if (srcInEZ && !dstInEZ) { throw new IOException( - src + " can't be moved from an encryption zone."); + srcIIP.getPath() + " can't be moved from an encryption zone."); } else if (dstInEZ && !srcInEZ) { throw new IOException( - src + " can't be moved into an encryption zone."); + srcIIP.getPath() + " can't be moved into an encryption zone."); } if (srcInEZ) { if (srcParentEZI != dstParentEZI) { final String srcEZPath = getFullPathName(srcParentEZI); final String dstEZPath = getFullPathName(dstParentEZI); - final StringBuilder sb = new StringBuilder(src); + final StringBuilder sb = new StringBuilder(srcIIP.getPath()); sb.append(" can't be moved from encryption zone "); sb.append(srcEZPath); sb.append(" to encryption zone "); @@ -300,15 +302,14 @@ public class EncryptionZoneManager { *

    * Called while holding the FSDirectory lock. */ - XAttr createEncryptionZone(String src, CipherSuite suite, + XAttr createEncryptionZone(INodesInPath srcIIP, CipherSuite suite, CryptoProtocolVersion version, String keyName) throws IOException { assert dir.hasWriteLock(); // Check if src is a valid path for new EZ creation - final INodesInPath srcIIP = dir.getINodesInPath4Write(src, false); - if (srcIIP == null || srcIIP.getLastINode() == null) { - throw new FileNotFoundException("cannot find " + src); + if (srcIIP.getLastINode() == null) { + throw new FileNotFoundException("cannot find " + srcIIP.getPath()); } if (dir.isNonEmptyDirectory(srcIIP)) { throw new IOException( @@ -322,8 +323,8 @@ public class EncryptionZoneManager { if (hasCreatedEncryptionZone() && encryptionZones. get(srcINode.getId()) != null) { - throw new IOException("Directory " + src + " is already an encryption " + - "zone."); + throw new IOException( + "Directory " + srcIIP.getPath() + " is already an encryption zone."); } final HdfsProtos.ZoneEncryptionInfoProto proto = @@ -335,7 +336,7 @@ public class EncryptionZoneManager { xattrs.add(ezXAttr); // updating the xattr will call addEncryptionZone, // done this way to handle edit log loading - FSDirXAttrOp.unprotectedSetXAttrs(dir, src, xattrs, + FSDirXAttrOp.unprotectedSetXAttrs(dir, srcIIP, xattrs, EnumSet.of(XAttrSetFlag.CREATE)); return ezXAttr; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java index 2153f02726c..afafd789bb7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java @@ -152,7 +152,6 @@ class FSDirAclOp { fsd.readLock(); try { INodesInPath iip = fsd.resolvePath(pc, src); - src = iip.getPath(); // There is no real inode for the path ending in ".snapshot", so return a // non-null, unpopulated AclStatus. This is similar to getFileInfo. if (iip.isDotSnapshotDir() && fsd.getINode4DotSnapshot(iip) != null) { @@ -163,8 +162,7 @@ class FSDirAclOp { } INode inode = FSDirectory.resolveLastINode(iip); int snapshotId = iip.getPathSnapshotId(); - List acl = AclStorage.readINodeAcl(fsd.getAttributes(src, - inode.getLocalNameBytes(), inode, snapshotId)); + List acl = AclStorage.readINodeAcl(fsd.getAttributes(iip)); FsPermission fsPermission = inode.getFsPermission(snapshotId); return new AclStatus.Builder() .owner(inode.getUserName()).group(inode.getGroupName()) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java index 4c5ecb1d226..91d9bce548b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java @@ -50,9 +50,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KE public class FSDirAttrOp { static HdfsFileStatus setPermission( - FSDirectory fsd, final String srcArg, FsPermission permission) + FSDirectory fsd, final String src, FsPermission permission) throws IOException { - String src = srcArg; if (FSDirectory.isExactReservedName(src)) { throw new InvalidPathException(src); } @@ -61,13 +60,12 @@ public class FSDirAttrOp { fsd.writeLock(); try { iip = fsd.resolvePathForWrite(pc, src); - src = iip.getPath(); fsd.checkOwner(pc, iip); - unprotectedSetPermission(fsd, src, permission); + unprotectedSetPermission(fsd, iip, permission); } finally { fsd.writeUnlock(); } - fsd.getEditLog().logSetPermissions(src, permission); + fsd.getEditLog().logSetPermissions(iip.getPath(), permission); return fsd.getAuditFileInfo(iip); } @@ -82,7 +80,6 @@ public class FSDirAttrOp { fsd.writeLock(); try { iip = fsd.resolvePathForWrite(pc, src); - src = iip.getPath(); fsd.checkOwner(pc, iip); if (!pc.isSuperUser()) { if (username != null && !pc.getUser().equals(username)) { @@ -92,11 +89,11 @@ public class FSDirAttrOp { throw new AccessControlException("User does not belong to " + group); } } - unprotectedSetOwner(fsd, src, username, group); + unprotectedSetOwner(fsd, iip, username, group); } finally { fsd.writeUnlock(); } - fsd.getEditLog().logSetOwner(src, username, group); + fsd.getEditLog().logSetOwner(iip.getPath(), username, group); return fsd.getAuditFileInfo(iip); } @@ -109,20 +106,18 @@ public class FSDirAttrOp { fsd.writeLock(); try { iip = fsd.resolvePathForWrite(pc, src); - src = iip.getPath(); // Write access is required to set access and modification times if (fsd.isPermissionEnabled()) { fsd.checkPathAccess(pc, iip, FsAction.WRITE); } final INode inode = iip.getLastINode(); if (inode == null) { - throw new FileNotFoundException("File/Directory " + src + + throw new FileNotFoundException("File/Directory " + iip.getPath() + " does not exist."); } - boolean changed = unprotectedSetTimes(fsd, inode, mtime, atime, true, - iip.getLatestSnapshotId()); + boolean changed = unprotectedSetTimes(fsd, iip, mtime, atime, true); if (changed) { - fsd.getEditLog().logTimes(src, mtime, atime); + fsd.getEditLog().logTimes(iip.getPath(), mtime, atime); } } finally { fsd.writeUnlock(); @@ -139,16 +134,15 @@ public class FSDirAttrOp { fsd.writeLock(); try { final INodesInPath iip = fsd.resolvePathForWrite(pc, src); - src = iip.getPath(); if (fsd.isPermissionEnabled()) { fsd.checkPathAccess(pc, iip, FsAction.WRITE); } - final BlockInfo[] blocks = unprotectedSetReplication(fsd, src, + final BlockInfo[] blocks = unprotectedSetReplication(fsd, iip, replication); isFile = blocks != null; if (isFile) { - fsd.getEditLog().logSetReplication(src, replication); + fsd.getEditLog().logSetReplication(iip.getPath(), replication); } } finally { fsd.writeUnlock(); @@ -186,15 +180,14 @@ public class FSDirAttrOp { INodesInPath iip; fsd.writeLock(); try { - src = FSDirectory.resolvePath(src, fsd); - iip = fsd.getINodesInPath4Write(src); + iip = fsd.resolvePathForWrite(pc, src); if (fsd.isPermissionEnabled()) { fsd.checkPathAccess(pc, iip, FsAction.WRITE); } unprotectedSetStoragePolicy(fsd, bm, iip, policyId); - fsd.getEditLog().logSetStoragePolicy(src, policyId); + fsd.getEditLog().logSetStoragePolicy(iip.getPath(), policyId); } finally { fsd.writeUnlock(); } @@ -232,11 +225,10 @@ public class FSDirAttrOp { fsd.readLock(); try { final INodesInPath iip = fsd.resolvePath(pc, src, false); - src = iip.getPath(); if (fsd.isPermissionEnabled()) { fsd.checkTraverse(pc, iip); } - return INodeFile.valueOf(iip.getLastINode(), src) + return INodeFile.valueOf(iip.getLastINode(), iip.getPath()) .getPreferredBlockSize(); } finally { fsd.readUnlock(); @@ -250,14 +242,16 @@ public class FSDirAttrOp { */ static void setQuota(FSDirectory fsd, String src, long nsQuota, long ssQuota, StorageType type) throws IOException { + FSPermissionChecker pc = fsd.getPermissionChecker(); if (fsd.isPermissionEnabled()) { - FSPermissionChecker pc = fsd.getPermissionChecker(); pc.checkSuperuserPrivilege(); } fsd.writeLock(); try { - INodeDirectory changed = unprotectedSetQuota(fsd, src, nsQuota, ssQuota, type); + INodesInPath iip = fsd.resolvePathForWrite(pc, src); + INodeDirectory changed = + unprotectedSetQuota(fsd, iip, nsQuota, ssQuota, type); if (changed != null) { final QuotaCounts q = changed.getQuotaCounts(); if (type == null) { @@ -273,58 +267,40 @@ public class FSDirAttrOp { } static void unprotectedSetPermission( - FSDirectory fsd, String src, FsPermission permissions) + FSDirectory fsd, INodesInPath iip, FsPermission permissions) throws FileNotFoundException, UnresolvedLinkException, QuotaExceededException, SnapshotAccessControlException { assert fsd.hasWriteLock(); - final INodesInPath inodesInPath = fsd.getINodesInPath4Write(src, true); - final INode inode = inodesInPath.getLastINode(); - if (inode == null) { - throw new FileNotFoundException("File does not exist: " + src); - } - int snapshotId = inodesInPath.getLatestSnapshotId(); + final INode inode = FSDirectory.resolveLastINode(iip); + int snapshotId = iip.getLatestSnapshotId(); inode.setPermission(permissions, snapshotId); } static void unprotectedSetOwner( - FSDirectory fsd, String src, String username, String groupname) + FSDirectory fsd, INodesInPath iip, String username, String groupname) throws FileNotFoundException, UnresolvedLinkException, QuotaExceededException, SnapshotAccessControlException { assert fsd.hasWriteLock(); - final INodesInPath inodesInPath = fsd.getINodesInPath4Write(src, true); - INode inode = inodesInPath.getLastINode(); - if (inode == null) { - throw new FileNotFoundException("File does not exist: " + src); - } + final INode inode = FSDirectory.resolveLastINode(iip); if (username != null) { - inode = inode.setUser(username, inodesInPath.getLatestSnapshotId()); + inode.setUser(username, iip.getLatestSnapshotId()); } if (groupname != null) { - inode.setGroup(groupname, inodesInPath.getLatestSnapshotId()); + inode.setGroup(groupname, iip.getLatestSnapshotId()); } } static boolean setTimes( - FSDirectory fsd, INode inode, long mtime, long atime, boolean force, - int latestSnapshotId) throws QuotaExceededException { + FSDirectory fsd, INodesInPath iip, long mtime, long atime, boolean force) + throws QuotaExceededException { fsd.writeLock(); try { - return unprotectedSetTimes(fsd, inode, mtime, atime, force, - latestSnapshotId); + return unprotectedSetTimes(fsd, iip, mtime, atime, force); } finally { fsd.writeUnlock(); } } - static boolean unprotectedSetTimes( - FSDirectory fsd, String src, long mtime, long atime, boolean force) - throws UnresolvedLinkException, QuotaExceededException { - assert fsd.hasWriteLock(); - final INodesInPath i = fsd.getINodesInPath(src, true); - return unprotectedSetTimes(fsd, i.getLastINode(), mtime, atime, - force, i.getLatestSnapshotId()); - } - /** * See {@link org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String, * long, long, StorageType)} @@ -339,7 +315,8 @@ public class FSDirAttrOp { * @throws SnapshotAccessControlException if path is in RO snapshot */ static INodeDirectory unprotectedSetQuota( - FSDirectory fsd, String src, long nsQuota, long ssQuota, StorageType type) + FSDirectory fsd, INodesInPath iip, long nsQuota, + long ssQuota, StorageType type) throws FileNotFoundException, PathIsNotDirectoryException, QuotaExceededException, UnresolvedLinkException, SnapshotAccessControlException, UnsupportedActionException { @@ -363,9 +340,8 @@ public class FSDirAttrOp { nsQuota); } - String srcs = FSDirectory.normalizePath(src); - final INodesInPath iip = fsd.getINodesInPath4Write(srcs, true); - INodeDirectory dirNode = INodeDirectory.valueOf(iip.getLastINode(), srcs); + INodeDirectory dirNode = + INodeDirectory.valueOf(iip.getLastINode(), iip.getPath()); if (dirNode.isRoot() && nsQuota == HdfsConstants.QUOTA_RESET) { throw new IllegalArgumentException("Cannot clear namespace quota on root."); } else { // a directory inode @@ -401,13 +377,12 @@ public class FSDirAttrOp { } static BlockInfo[] unprotectedSetReplication( - FSDirectory fsd, String src, short replication) + FSDirectory fsd, INodesInPath iip, short replication) throws QuotaExceededException, UnresolvedLinkException, SnapshotAccessControlException, UnsupportedActionException { assert fsd.hasWriteLock(); final BlockManager bm = fsd.getBlockManager(); - final INodesInPath iip = fsd.getINodesInPath4Write(src, true); final INode inode = iip.getLastINode(); if (inode == null || !inode.isFile() || inode.asFile().isStriped()) { // TODO we do not support replication on stripe layout files yet @@ -438,10 +413,10 @@ public class FSDirAttrOp { if (oldBR != -1) { if (oldBR > targetReplication) { FSDirectory.LOG.info("Decreasing replication from {} to {} for {}", - oldBR, targetReplication, src); + oldBR, targetReplication, iip.getPath()); } else { FSDirectory.LOG.info("Increasing replication from {} to {} for {}", - oldBR, targetReplication, src); + oldBR, targetReplication, iip.getPath()); } } return file.getBlocks(); @@ -476,8 +451,7 @@ public class FSDirAttrOp { } inode.asFile().setStoragePolicyID(policyId, snapshotId); } else if (inode.isDirectory()) { - setDirStoragePolicy(fsd, inode.asDirectory(), policyId, - snapshotId); + setDirStoragePolicy(fsd, iip, policyId); } else { throw new FileNotFoundException(iip.getPath() + " is not a file or directory"); @@ -485,8 +459,8 @@ public class FSDirAttrOp { } private static void setDirStoragePolicy( - FSDirectory fsd, INodeDirectory inode, byte policyId, - int latestSnapshotId) throws IOException { + FSDirectory fsd, INodesInPath iip, byte policyId) throws IOException { + INode inode = FSDirectory.resolveLastINode(iip); List existingXAttrs = XAttrStorage.readINodeXAttrs(inode); XAttr xAttr = BlockStoragePolicySuite.buildXAttr(policyId); List newXAttrs = null; @@ -501,14 +475,16 @@ public class FSDirAttrOp { Arrays.asList(xAttr), EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); } - XAttrStorage.updateINodeXAttrs(inode, newXAttrs, latestSnapshotId); + XAttrStorage.updateINodeXAttrs(inode, newXAttrs, iip.getLatestSnapshotId()); } - private static boolean unprotectedSetTimes( - FSDirectory fsd, INode inode, long mtime, long atime, boolean force, - int latest) throws QuotaExceededException { + static boolean unprotectedSetTimes( + FSDirectory fsd, INodesInPath iip, long mtime, long atime, boolean force) + throws QuotaExceededException { assert fsd.hasWriteLock(); boolean status = false; + INode inode = iip.getLastINode(); + int latest = iip.getLatestSnapshotId(); if (mtime != -1) { inode = inode.setModificationTime(mtime, latest); status = true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java index 13f109201be..328ce799ef1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java @@ -55,9 +55,9 @@ class FSDirDeleteOp { FSNamesystem fsn = fsd.getFSNamesystem(); fsd.writeLock(); try { - if (deleteAllowed(iip, iip.getPath()) ) { + if (deleteAllowed(iip)) { List snapshottableDirs = new ArrayList<>(); - FSDirSnapshotOp.checkSnapshot(iip.getLastINode(), snapshottableDirs); + FSDirSnapshotOp.checkSnapshot(fsd, iip, snapshottableDirs); ReclaimContext context = new ReclaimContext( fsd.getBlockStoragePolicySuite(), collectedBlocks, removedINodes, removedUCFiles); @@ -98,20 +98,24 @@ class FSDirDeleteOp { FSDirectory fsd = fsn.getFSDirectory(); FSPermissionChecker pc = fsd.getPermissionChecker(); - final INodesInPath iip = fsd.resolvePathForWrite(pc, src, false); - src = iip.getPath(); - if (!recursive && fsd.isNonEmptyDirectory(iip)) { - throw new PathIsNotEmptyDirectoryException(src + " is non empty"); + if (FSDirectory.isExactReservedName(src)) { + throw new InvalidPathException(src); } + + final INodesInPath iip = fsd.resolvePathForWrite(pc, src, false); if (fsd.isPermissionEnabled()) { fsd.checkPermission(pc, iip, false, null, FsAction.WRITE, null, FsAction.ALL, true); } - if (recursive && fsd.isNonEmptyDirectory(iip)) { - checkProtectedDescendants(fsd, src); + if (fsd.isNonEmptyDirectory(iip)) { + if (!recursive) { + throw new PathIsNotEmptyDirectoryException( + iip.getPath() + " is non empty"); + } + checkProtectedDescendants(fsd, iip); } - return deleteInternal(fsn, src, iip, logRetryCache); + return deleteInternal(fsn, iip, logRetryCache); } /** @@ -126,21 +130,18 @@ class FSDirDeleteOp { * @param src a string representation of a path to an inode * @param mtime the time the inode is removed */ - static void deleteForEditLog(FSDirectory fsd, String src, long mtime) + static void deleteForEditLog(FSDirectory fsd, INodesInPath iip, long mtime) throws IOException { assert fsd.hasWriteLock(); FSNamesystem fsn = fsd.getFSNamesystem(); BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo(); List removedINodes = new ChunkedArrayList<>(); List removedUCFiles = new ChunkedArrayList<>(); - - final INodesInPath iip = fsd.getINodesInPath4Write( - FSDirectory.normalizePath(src), false); - if (!deleteAllowed(iip, src)) { + if (!deleteAllowed(iip)) { return; } List snapshottableDirs = new ArrayList<>(); - FSDirSnapshotOp.checkSnapshot(iip.getLastINode(), snapshottableDirs); + FSDirSnapshotOp.checkSnapshot(fsd, iip, snapshottableDirs); boolean filesRemoved = unprotectedDelete(fsd, iip, new ReclaimContext(fsd.getBlockStoragePolicySuite(), collectedBlocks, removedINodes, removedUCFiles), @@ -162,7 +163,6 @@ class FSDirDeleteOp { *

    * For small directory or file the deletion is done in one shot. * @param fsn namespace - * @param src path name to be deleted * @param iip the INodesInPath instance containing all the INodes for the path * @param logRetryCache whether to record RPC ids in editlog for retry cache * rebuilding @@ -170,15 +170,11 @@ class FSDirDeleteOp { * @throws IOException */ static BlocksMapUpdateInfo deleteInternal( - FSNamesystem fsn, String src, INodesInPath iip, boolean logRetryCache) + FSNamesystem fsn, INodesInPath iip, boolean logRetryCache) throws IOException { assert fsn.hasWriteLock(); if (NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("DIR* NameSystem.delete: " + src); - } - - if (FSDirectory.isExactReservedName(src)) { - throw new InvalidPathException(src); + NameNode.stateChangeLog.debug("DIR* NameSystem.delete: " + iip.getPath()); } FSDirectory fsd = fsn.getFSDirectory(); @@ -193,14 +189,14 @@ class FSDirDeleteOp { if (filesRemoved < 0) { return null; } - fsd.getEditLog().logDelete(src, mtime, logRetryCache); + fsd.getEditLog().logDelete(iip.getPath(), mtime, logRetryCache); incrDeletedFileCount(filesRemoved); fsn.removeLeasesAndINodes(removedUCFiles, removedINodes, true); if (NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("DIR* Namesystem.delete: " - + src +" is removed"); + NameNode.stateChangeLog.debug( + "DIR* Namesystem.delete: " + iip.getPath() +" is removed"); } return collectedBlocks; } @@ -209,19 +205,18 @@ class FSDirDeleteOp { NameNode.getNameNodeMetrics().incrFilesDeleted(count); } - private static boolean deleteAllowed(final INodesInPath iip, - final String src) { + private static boolean deleteAllowed(final INodesInPath iip) { if (iip.length() < 1 || iip.getLastINode() == null) { if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug( "DIR* FSDirectory.unprotectedDelete: failed to remove " - + src + " because it does not exist"); + + iip.getPath() + " because it does not exist"); } return false; } else if (iip.length() == 1) { // src is the root NameNode.stateChangeLog.warn( - "DIR* FSDirectory.unprotectedDelete: failed to remove " + src + - " because the root is not allowed to be deleted"); + "DIR* FSDirectory.unprotectedDelete: failed to remove " + + iip.getPath() + " because the root is not allowed to be deleted"); return false; } return true; @@ -278,15 +273,19 @@ class FSDirDeleteOp { * Throw if the given directory has any non-empty protected descendants * (including itself). * - * @param src directory whose descendants are to be checked. The caller - * must ensure src is not terminated with {@link Path#SEPARATOR}. + * @param iip directory whose descendants are to be checked. * @throws AccessControlException if a non-empty protected descendant * was found. */ - private static void checkProtectedDescendants(FSDirectory fsd, String src) - throws AccessControlException, UnresolvedLinkException { + private static void checkProtectedDescendants( + FSDirectory fsd, INodesInPath iip) + throws AccessControlException, UnresolvedLinkException { final SortedSet protectedDirs = fsd.getProtectedDirectories(); + if (protectedDirs.isEmpty()) { + return; + } + String src = iip.getPath(); // Is src protected? Caller has already checked it is non-empty. if (protectedDirs.contains(src)) { throw new AccessControlException( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java index 7501fc39005..d7a36112781 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO; -import java.io.FileNotFoundException; import java.io.IOException; import java.security.GeneralSecurityException; import java.security.PrivilegedExceptionAction; @@ -73,8 +72,11 @@ final class FSDirEncryptionZoneOp { * @return New EDEK, or null if ezKeyName is null * @throws IOException */ - static EncryptedKeyVersion generateEncryptedDataEncryptionKey( + private static EncryptedKeyVersion generateEncryptedDataEncryptionKey( final FSDirectory fsd, final String ezKeyName) throws IOException { + // must not be holding lock during this operation + assert !fsd.getFSNamesystem().hasReadLock(); + assert !fsd.getFSNamesystem().hasWriteLock(); if (ezKeyName == null) { return null; } @@ -148,23 +150,21 @@ final class FSDirEncryptionZoneOp { final String keyName, final boolean logRetryCache) throws IOException { final CipherSuite suite = CipherSuite.convert(cipher); List xAttrs = Lists.newArrayListWithCapacity(1); - final String src; // For now this is hard coded, as we only support one method. final CryptoProtocolVersion version = CryptoProtocolVersion.ENCRYPTION_ZONES; + final INodesInPath iip; fsd.writeLock(); try { - final INodesInPath iip = fsd.resolvePath(pc, srcArg); - src = iip.getPath(); - final XAttr ezXAttr = fsd.ezManager.createEncryptionZone(src, suite, + iip = fsd.resolvePathForWrite(pc, srcArg); + final XAttr ezXAttr = fsd.ezManager.createEncryptionZone(iip, suite, version, keyName); xAttrs.add(ezXAttr); } finally { fsd.writeUnlock(); } - fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache); - final INodesInPath iip = fsd.getINodesInPath4Write(src, false); + fsd.getEditLog().logSetXAttrs(iip.getPath(), xAttrs, logRetryCache); return fsd.getAuditFileInfo(iip); } @@ -184,9 +184,6 @@ final class FSDirEncryptionZoneOp { fsd.readLock(); try { iip = fsd.resolvePath(pc, srcArg); - if (iip.getLastINode() == null) { - throw new FileNotFoundException("Path not found: " + iip.getPath()); - } if (fsd.isPermissionEnabled()) { fsd.checkPathAccess(pc, iip, FsAction.READ); } @@ -227,8 +224,9 @@ final class FSDirEncryptionZoneOp { * @param info file encryption information * @throws IOException */ - static void setFileEncryptionInfo(final FSDirectory fsd, final String src, - final FileEncryptionInfo info) throws IOException { + static void setFileEncryptionInfo(final FSDirectory fsd, + final INodesInPath iip, final FileEncryptionInfo info) + throws IOException { // Make the PB for the xattr final HdfsProtos.PerFileEncryptionInfoProto proto = PBHelperClient.convertPerFileEncInfo(info); @@ -239,7 +237,7 @@ final class FSDirEncryptionZoneOp { xAttrs.add(fileEncryptionAttr); fsd.writeLock(); try { - FSDirXAttrOp.unprotectedSetXAttrs(fsd, src, xAttrs, + FSDirXAttrOp.unprotectedSetXAttrs(fsd, iip, xAttrs, EnumSet.of(XAttrSetFlag.CREATE)); } finally { fsd.writeUnlock(); @@ -250,21 +248,18 @@ final class FSDirEncryptionZoneOp { * This function combines the per-file encryption info (obtained * from the inode's XAttrs), and the encryption info from its zone, and * returns a consolidated FileEncryptionInfo instance. Null is returned - * for non-encrypted files. + * for non-encrypted or raw files. * * @param fsd fsdirectory - * @param inode inode of the file - * @param snapshotId ID of the snapshot that - * we want to get encryption info from * @param iip inodes in the path containing the file, passed in to - * avoid obtaining the list of inodes again; if iip is - * null then the list of inodes will be obtained again + * avoid obtaining the list of inodes again * @return consolidated file encryption info; null for non-encrypted files */ static FileEncryptionInfo getFileEncryptionInfo(final FSDirectory fsd, - final INode inode, final int snapshotId, final INodesInPath iip) - throws IOException { - if (!inode.isFile() || !fsd.ezManager.hasCreatedEncryptionZone()) { + final INodesInPath iip) throws IOException { + if (iip.isRaw() || + !fsd.ezManager.hasCreatedEncryptionZone() || + !iip.getLastINode().isFile()) { return null; } fsd.readLock(); @@ -284,8 +279,8 @@ final class FSDirEncryptionZoneOp { final CryptoProtocolVersion version = encryptionZone.getVersion(); final CipherSuite suite = encryptionZone.getSuite(); final String keyName = encryptionZone.getKeyName(); - XAttr fileXAttr = FSDirXAttrOp.unprotectedGetXAttrByPrefixedName(inode, - snapshotId, CRYPTO_XATTR_FILE_ENCRYPTION_INFO); + XAttr fileXAttr = FSDirXAttrOp.unprotectedGetXAttrByPrefixedName( + iip, CRYPTO_XATTR_FILE_ENCRYPTION_INFO); if (fileXAttr == null) { NameNode.LOG.warn("Could not find encryption XAttr for file " + @@ -299,15 +294,53 @@ final class FSDirEncryptionZoneOp { return PBHelperClient.convert(fileProto, suite, version, keyName); } catch (InvalidProtocolBufferException e) { throw new IOException("Could not parse file encryption info for " + - "inode " + inode, e); + "inode " + iip.getPath(), e); } } finally { fsd.readUnlock(); } } + /** + * If the file and encryption key are valid, return the encryption info, + * else throw a retry exception. The startFile method generates the EDEK + * outside of the lock so the zone must be reverified. + * + * @param dir fsdirectory + * @param iip inodes in the file path + * @param ezInfo the encryption key + * @return FileEncryptionInfo for the file + * @throws RetryStartFileException if key is inconsistent with current zone + */ + static FileEncryptionInfo getFileEncryptionInfo(FSDirectory dir, + INodesInPath iip, EncryptionKeyInfo ezInfo) + throws RetryStartFileException { + FileEncryptionInfo feInfo = null; + final EncryptionZone zone = getEZForPath(dir, iip); + if (zone != null) { + // The path is now within an EZ, but we're missing encryption parameters + if (ezInfo == null) { + throw new RetryStartFileException(); + } + // Path is within an EZ and we have provided encryption parameters. + // Make sure that the generated EDEK matches the settings of the EZ. + final String ezKeyName = zone.getKeyName(); + if (!ezKeyName.equals(ezInfo.edek.getEncryptionKeyName())) { + throw new RetryStartFileException(); + } + feInfo = new FileEncryptionInfo(ezInfo.suite, ezInfo.protocolVersion, + ezInfo.edek.getEncryptedKeyVersion().getMaterial(), + ezInfo.edek.getEncryptedKeyIv(), + ezKeyName, ezInfo.edek.getEncryptionKeyVersionName()); + } + return feInfo; + } + static boolean isInAnEZ(final FSDirectory fsd, final INodesInPath iip) throws UnresolvedLinkException, SnapshotAccessControlException { + if (!fsd.ezManager.hasCreatedEncryptionZone()) { + return false; + } fsd.readLock(); try { return fsd.ezManager.isInAnEZ(iip); @@ -403,4 +436,67 @@ final class FSDirEncryptionZoneOp { } } } + + /** + * If the file is in an encryption zone, we optimistically create an + * EDEK for the file by calling out to the configured KeyProvider. + * Since this typically involves doing an RPC, the fsn lock is yielded. + * + * Since the path can flip-flop between being in an encryption zone and not + * in the meantime, the call MUST re-resolve the IIP and re-check + * preconditions if this method does not return null; + * + * @param fsn the namesystem. + * @param iip the inodes for the path + * @param supportedVersions client's supported versions + * @return EncryptionKeyInfo if the path is in an EZ, else null + */ + static EncryptionKeyInfo getEncryptionKeyInfo(FSNamesystem fsn, + INodesInPath iip, CryptoProtocolVersion[] supportedVersions) + throws IOException { + FSDirectory fsd = fsn.getFSDirectory(); + // Nothing to do if the path is not within an EZ + final EncryptionZone zone = getEZForPath(fsd, iip); + if (zone == null) { + EncryptionFaultInjector.getInstance().startFileNoKey(); + return null; + } + CryptoProtocolVersion protocolVersion = fsn.chooseProtocolVersion( + zone, supportedVersions); + CipherSuite suite = zone.getSuite(); + String ezKeyName = zone.getKeyName(); + + Preconditions.checkNotNull(protocolVersion); + Preconditions.checkNotNull(suite); + Preconditions.checkArgument(!suite.equals(CipherSuite.UNKNOWN), + "Chose an UNKNOWN CipherSuite!"); + Preconditions.checkNotNull(ezKeyName); + + // Generate EDEK while not holding the fsn lock. + fsn.writeUnlock(); + try { + EncryptionFaultInjector.getInstance().startFileBeforeGenerateKey(); + return new EncryptionKeyInfo(protocolVersion, suite, ezKeyName, + generateEncryptedDataEncryptionKey(fsd, ezKeyName)); + } finally { + fsn.writeLock(); + EncryptionFaultInjector.getInstance().startFileAfterGenerateKey(); + } + } + + static class EncryptionKeyInfo { + final CryptoProtocolVersion protocolVersion; + final CipherSuite suite; + final String ezKeyName; + final KeyProviderCryptoExtension.EncryptedKeyVersion edek; + + EncryptionKeyInfo( + CryptoProtocolVersion protocolVersion, CipherSuite suite, + String ezKeyName, KeyProviderCryptoExtension.EncryptedKeyVersion edek) { + this.protocolVersion = protocolVersion; + this.suite = suite; + this.ezKeyName = ezKeyName; + this.edek = edek; + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 17544f53850..25b3155e04f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -141,7 +141,7 @@ final class FSDirErasureCodingOp { } final List xattrs = Lists.newArrayListWithCapacity(1); xattrs.add(ecXAttr); - FSDirXAttrOp.unprotectedSetXAttrs(fsd, src, xattrs, + FSDirXAttrOp.unprotectedSetXAttrs(fsd, srcIIP, xattrs, EnumSet.of(XAttrSetFlag.CREATE)); return xattrs; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java index 2d1914f1625..4d8d7d7a8a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java @@ -66,7 +66,7 @@ class FSDirMkdirOp { } if (!createParent) { - fsd.verifyParentDir(iip, src); + fsd.verifyParentDir(iip); } // validate that we have enough inodes. This is, at best, a diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java index 0fdc545cfcb..12d5cfead83 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java @@ -156,7 +156,7 @@ class FSDirRenameOp { assert fsd.hasWriteLock(); final INode srcInode = srcIIP.getLastINode(); try { - validateRenameSource(srcIIP); + validateRenameSource(fsd, srcIIP); } catch (SnapshotException e) { throw e; } catch (IOException ignored) { @@ -190,7 +190,7 @@ class FSDirRenameOp { return null; } - fsd.ezManager.checkMoveValidity(srcIIP, dstIIP, src); + fsd.ezManager.checkMoveValidity(srcIIP, dstIIP); // Ensure dst has quota to accommodate rename verifyFsLimitsForRename(fsd, srcIIP, dstIIP); verifyQuotaForRename(fsd, srcIIP, dstIIP); @@ -365,7 +365,7 @@ class FSDirRenameOp { final String dst = dstIIP.getPath(); final String error; final INode srcInode = srcIIP.getLastINode(); - validateRenameSource(srcIIP); + validateRenameSource(fsd, srcIIP); // validate the destination if (dst.equals(src)) { @@ -382,12 +382,12 @@ class FSDirRenameOp { } BlockStoragePolicySuite bsps = fsd.getBlockStoragePolicySuite(); - fsd.ezManager.checkMoveValidity(srcIIP, dstIIP, src); + fsd.ezManager.checkMoveValidity(srcIIP, dstIIP); final INode dstInode = dstIIP.getLastINode(); List snapshottableDirs = new ArrayList<>(); if (dstInode != null) { // Destination exists validateOverwrite(src, dst, overwrite, srcInode, dstInode); - FSDirSnapshotOp.checkSnapshot(dstInode, snapshottableDirs); + FSDirSnapshotOp.checkSnapshot(fsd, dstIIP, snapshottableDirs); } INode dstParent = dstIIP.getINode(-2); @@ -559,8 +559,8 @@ class FSDirRenameOp { } } - private static void validateRenameSource(INodesInPath srcIIP) - throws IOException { + private static void validateRenameSource(FSDirectory fsd, + INodesInPath srcIIP) throws IOException { String error; final INode srcInode = srcIIP.getLastINode(); // validate source @@ -578,7 +578,7 @@ class FSDirRenameOp { } // srcInode and its subtree cannot contain snapshottable directories with // snapshots - FSDirSnapshotOp.checkSnapshot(srcInode, null); + FSDirSnapshotOp.checkSnapshot(fsd, srcIIP, null); } private static class RenameOperation { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java index c565a6aac00..ad282d164d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java @@ -35,7 +35,6 @@ import org.apache.hadoop.util.ChunkedArrayList; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.ListIterator; import java.util.List; class FSDirSnapshotOp { @@ -252,7 +251,7 @@ class FSDirSnapshotOp { * @param snapshottableDirs The list of directories that are snapshottable * but do not have snapshots yet */ - static void checkSnapshot( + private static void checkSnapshot( INode target, List snapshottableDirs) throws SnapshotException { if (target.isDirectory()) { @@ -276,4 +275,23 @@ class FSDirSnapshotOp { } } } + + /** + * Check if the given path (or one of its descendants) is snapshottable and + * already has snapshots. + * + * @param fsd the FSDirectory + * @param iip inodes of the path + * @param snapshottableDirs The list of directories that are snapshottable + * but do not have snapshots yet + */ + static void checkSnapshot(FSDirectory fsd, INodesInPath iip, + List snapshottableDirs) throws SnapshotException { + // avoid the performance penalty of recursing the tree if snapshots + // are not in use + SnapshotManager sm = fsd.getFSNamesystem().getSnapshotManager(); + if (sm.getNumSnapshottableDirs() > 0) { + checkSnapshot(iip.getLastINode(), snapshottableDirs); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index 5072d68ad8b..5aa4dbc5ad0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hdfs.protocol.FsPermissionExtension; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; @@ -53,15 +52,12 @@ import static org.apache.hadoop.util.Time.now; class FSDirStatAndListingOp { static DirectoryListing getListingInt(FSDirectory fsd, final String srcArg, byte[] startAfter, boolean needLocation) throws IOException { - String src = null; - final INodesInPath iip; if (fsd.isPermissionEnabled()) { FSPermissionChecker pc = fsd.getPermissionChecker(); iip = fsd.resolvePath(pc, srcArg); - src = iip.getPath(); } else { - src = FSDirectory.resolvePath(srcArg, fsd); + String src = FSDirectory.resolvePath(srcArg, fsd); iip = fsd.getINodesInPath(src, true); } @@ -92,7 +88,7 @@ class FSDirStatAndListingOp { } isSuperUser = pc.isSuperUser(); } - return getListing(fsd, iip, src, startAfter, needLocation, isSuperUser); + return getListing(fsd, iip, startAfter, needLocation, isSuperUser); } /** @@ -159,9 +155,7 @@ class FSDirStatAndListingOp { "Negative offset is not supported. File: " + src); Preconditions.checkArgument(length >= 0, "Negative length is not supported. File: " + src); - CacheManager cm = fsd.getFSNamesystem().getCacheManager(); BlockManager bm = fsd.getBlockManager(); - boolean isReservedName = FSDirectory.isReservedRawName(src); fsd.readLock(); try { final INodesInPath iip = fsd.resolvePath(pc, src); @@ -169,7 +163,7 @@ class FSDirStatAndListingOp { final INodeFile inode = INodeFile.valueOf(iip.getLastINode(), src); if (fsd.isPermissionEnabled()) { fsd.checkPathAccess(pc, iip, FsAction.READ); - fsd.checkUnreadableBySuperuser(pc, inode, iip.getPathSnapshotId()); + fsd.checkUnreadableBySuperuser(pc, iip); } final long fileSize = iip.isSnapshot() @@ -184,9 +178,8 @@ class FSDirStatAndListingOp { isUc = false; } - final FileEncryptionInfo feInfo = isReservedName ? null - : FSDirEncryptionZoneOp.getFileEncryptionInfo(fsd, inode, - iip.getPathSnapshotId(), iip); + final FileEncryptionInfo feInfo = + FSDirEncryptionZoneOp.getFileEncryptionInfo(fsd, iip); final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp. getErasureCodingPolicy(fsd.getFSNamesystem(), iip); @@ -194,11 +187,6 @@ class FSDirStatAndListingOp { inode.getBlocks(iip.getPathSnapshotId()), fileSize, isUc, offset, length, needBlockToken, iip.isSnapshot(), feInfo, ecPolicy); - // Set caching information for the located blocks. - for (LocatedBlock lb : blocks.getLocatedBlocks()) { - cm.setCachedLocations(lb); - } - final long now = now(); boolean updateAccessTime = fsd.isAccessTimeSupported() && !iip.isSnapshot() @@ -225,42 +213,39 @@ class FSDirStatAndListingOp { * @param fsd FSDirectory * @param iip the INodesInPath instance containing all the INodes along the * path - * @param src the directory name * @param startAfter the name to start listing after * @param needLocation if block locations are returned + * @param includeStoragePolicy if storage policy is returned * @return a partial listing starting after startAfter */ private static DirectoryListing getListing(FSDirectory fsd, INodesInPath iip, - String src, byte[] startAfter, boolean needLocation, boolean isSuperUser) + byte[] startAfter, boolean needLocation, boolean includeStoragePolicy) throws IOException { - String srcs = FSDirectory.normalizePath(src); - if (FSDirectory.isExactReservedName(srcs)) { + if (FSDirectory.isExactReservedName(iip.getPathComponents())) { return getReservedListing(fsd); } fsd.readLock(); try { - if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) { - return getSnapshotsListing(fsd, srcs, startAfter); + if (iip.isDotSnapshotDir()) { + return getSnapshotsListing(fsd, iip, startAfter); } final int snapshot = iip.getPathSnapshotId(); final INode targetNode = iip.getLastINode(); - if (targetNode == null) + if (targetNode == null) { return null; - byte parentStoragePolicy = isSuperUser ? - targetNode.getStoragePolicyID() : HdfsConstants - .BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; + } + + byte parentStoragePolicy = includeStoragePolicy + ? targetNode.getStoragePolicyID() + : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; if (!targetNode.isDirectory()) { // return the file's status. note that the iip already includes the // target INode - INodeAttributes nodeAttrs = getINodeAttributes( - fsd, src, HdfsFileStatus.EMPTY_NAME, targetNode, - snapshot); return new DirectoryListing( new HdfsFileStatus[]{ createFileStatus( - fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs, - needLocation, parentStoragePolicy, iip) + fsd, iip, null, parentStoragePolicy, needLocation) }, 0); } @@ -274,20 +259,15 @@ class FSDirStatAndListingOp { int listingCnt = 0; HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing]; for (int i = 0; i < numOfListing && locationBudget > 0; i++) { - INode cur = contents.get(startChild+i); - byte curPolicy = isSuperUser && !cur.isSymlink()? - cur.getLocalStoragePolicyID(): - HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; - INodeAttributes nodeAttrs = getINodeAttributes( - fsd, src, cur.getLocalNameBytes(), cur, - snapshot); - final INodesInPath iipWithChild = INodesInPath.append(iip, cur, - cur.getLocalNameBytes()); - listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), nodeAttrs, - needLocation, getStoragePolicyID(curPolicy, parentStoragePolicy), - iipWithChild); + INode child = contents.get(startChild+i); + byte childStoragePolicy = (includeStoragePolicy && !child.isSymlink()) + ? getStoragePolicyID(child.getLocalStoragePolicyID(), + parentStoragePolicy) + : parentStoragePolicy; + listing[i] = + createFileStatus(fsd, iip, child, childStoragePolicy, needLocation); listingCnt++; - if (needLocation) { + if (listing[i] instanceof HdfsLocatedFileStatus) { // Once we hit lsLimit locations, stop. // This helps to prevent excessively large response payloads. // Approximate #locations with locatedBlockCount() * repl_factor @@ -312,17 +292,16 @@ class FSDirStatAndListingOp { * Get a listing of all the snapshots of a snapshottable directory */ private static DirectoryListing getSnapshotsListing( - FSDirectory fsd, String src, byte[] startAfter) + FSDirectory fsd, INodesInPath iip, byte[] startAfter) throws IOException { Preconditions.checkState(fsd.hasReadLock()); - Preconditions.checkArgument( - src.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR), - "%s does not end with %s", src, HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR); - - final String dirPath = FSDirectory.normalizePath(src.substring(0, - src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length())); - - final INode node = fsd.getINode(dirPath); + Preconditions.checkArgument(iip.isDotSnapshotDir(), + "%s does not end with %s", + iip.getPath(), HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR); + // drop off the null .snapshot component + iip = iip.getParentINodesInPath(); + final String dirPath = iip.getPath(); + final INode node = iip.getLastINode(); final INodeDirectory dirNode = INodeDirectory.valueOf(node, dirPath); final DirectorySnapshottableFeature sf = dirNode.getDirectorySnapshottableFeature(); if (sf == null) { @@ -336,13 +315,8 @@ class FSDirStatAndListingOp { final HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing]; for (int i = 0; i < numOfListing; i++) { Snapshot.Root sRoot = snapshots.get(i + skipSize).getRoot(); - INodeAttributes nodeAttrs = getINodeAttributes( - fsd, src, sRoot.getLocalNameBytes(), - node, Snapshot.CURRENT_STATE_ID); - listing[i] = createFileStatus( - fsd, sRoot.getLocalNameBytes(), nodeAttrs, - HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, - INodesInPath.fromINode(sRoot)); + listing[i] = createFileStatus(fsd, iip, sRoot, + HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, false); } return new DirectoryListing( listing, snapshots.size() - skipSize - numOfListing); @@ -360,7 +334,6 @@ class FSDirStatAndListingOp { /** Get the file info for a specific file. * @param fsd FSDirectory * @param iip The path to the file, the file is included - * @param isRawPath true if a /.reserved/raw pathname was passed by the user * @param includeStoragePolicy whether to include storage policy * @return object containing information regarding the file * or null if file not found @@ -373,15 +346,10 @@ class FSDirStatAndListingOp { if (node == null) { return null; } - - byte policyId = includeStoragePolicy && !node.isSymlink() ? - node.getStoragePolicyID() : - HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; - INodeAttributes nodeAttrs = getINodeAttributes(fsd, iip.getPath(), - HdfsFileStatus.EMPTY_NAME, - node, iip.getPathSnapshotId()); - return createFileStatus(fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs, - policyId, iip); + byte policy = (includeStoragePolicy && !node.isSymlink()) + ? node.getStoragePolicyID() + : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; + return createFileStatus(fsd, iip, null, policy, false); } finally { fsd.readUnlock(); } @@ -408,58 +376,50 @@ class FSDirStatAndListingOp { } /** - * create an hdfs file status from an inode + * create a hdfs file status from an iip. + * @param fsd FSDirectory + * @param iip The INodesInPath containing the INodeFile and its ancestors + * @return HdfsFileStatus without locations or storage policy + */ + static HdfsFileStatus createFileStatusForEditLog( + FSDirectory fsd, INodesInPath iip) throws IOException { + return createFileStatus(fsd, iip, + null, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, false); + } + + /** + * create a hdfs file status from an iip. * * @param fsd FSDirectory - * @param path the local name + * @param iip The INodesInPath containing the INodeFile and its ancestors. + * @param child for a directory listing of the iip, else null + * @param storagePolicy for the path or closest ancestor * @param needLocation if block locations need to be included or not - * @param isRawPath true if this is being called on behalf of a path in - * /.reserved/raw - * @param iip the INodesInPath containing the target INode and its ancestors + * @param includeStoragePolicy if storage policy should be returned * @return a file status * @throws java.io.IOException if any error occurs */ private static HdfsFileStatus createFileStatus( - FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs, - boolean needLocation, byte storagePolicy, INodesInPath iip) - throws IOException { - if (needLocation) { - return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy, iip); - } else { - return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip); + FSDirectory fsd, INodesInPath iip, INode child, byte storagePolicy, + boolean needLocation) throws IOException { + assert fsd.hasReadLock(); + // only directory listing sets the status name. + byte[] name = HdfsFileStatus.EMPTY_NAME; + if (child != null) { + name = child.getLocalNameBytes(); + // have to do this for EC and EZ lookups... + iip = INodesInPath.append(iip, child, name); } - } - /** - * Create FileStatus for an given INodeFile. - * @param iip The INodesInPath containing the INodeFile and its ancestors - */ - static HdfsFileStatus createFileStatusForEditLog( - FSDirectory fsd, String fullPath, byte[] path, - byte storagePolicy, int snapshot, boolean isRawPath, - INodesInPath iip) throws IOException { - INodeAttributes nodeAttrs = getINodeAttributes( - fsd, fullPath, path, iip.getLastINode(), snapshot); - return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip); - } - - /** - * create file status for a given INode - * @param iip the INodesInPath containing the target INode and its ancestors - */ - static HdfsFileStatus createFileStatus( - FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs, - byte storagePolicy, INodesInPath iip) throws IOException { long size = 0; // length is zero for directories short replication = 0; long blocksize = 0; - final boolean isEncrypted; final INode node = iip.getLastINode(); final int snapshot = iip.getPathSnapshotId(); - final boolean isRawPath = iip.isRaw(); + LocatedBlocks loc = null; - final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp - .getFileEncryptionInfo(fsd, node, snapshot, iip); + final boolean isEncrypted = FSDirEncryptionZoneOp.isInAnEZ(fsd, iip); + FileEncryptionInfo feInfo = null; final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp .getErasureCodingPolicy(fsd.getFSNamesystem(), iip); @@ -469,16 +429,28 @@ class FSDirStatAndListingOp { size = fileNode.computeFileSize(snapshot); replication = fileNode.getFileReplication(snapshot); blocksize = fileNode.getPreferredBlockSize(); - isEncrypted = (feInfo != null) - || (isRawPath && FSDirEncryptionZoneOp.isInAnEZ(fsd, iip)); - } else { - isEncrypted = FSDirEncryptionZoneOp.isInAnEZ(fsd, iip); + if (isEncrypted) { + feInfo = FSDirEncryptionZoneOp.getFileEncryptionInfo(fsd, iip); + } + if (needLocation) { + final boolean inSnapshot = snapshot != Snapshot.CURRENT_STATE_ID; + final boolean isUc = !inSnapshot && fileNode.isUnderConstruction(); + final long fileSize = !inSnapshot && isUc + ? fileNode.computeFileSizeNotIncludingLastUcBlock() : size; + loc = fsd.getBlockManager().createLocatedBlocks( + fileNode.getBlocks(snapshot), fileSize, isUc, 0L, size, false, + inSnapshot, feInfo, ecPolicy); + if (loc == null) { + loc = new LocatedBlocks(); + } + } } int childrenNum = node.isDirectory() ? node.asDirectory().getChildrenNum(snapshot) : 0; - return new HdfsFileStatus( + INodeAttributes nodeAttrs = fsd.getAttributes(iip); + return createFileStatus( size, node.isDirectory(), replication, @@ -489,81 +461,30 @@ class FSDirStatAndListingOp { nodeAttrs.getUserName(), nodeAttrs.getGroupName(), node.isSymlink() ? node.asSymlink().getSymlink() : null, - path, + name, node.getId(), childrenNum, feInfo, storagePolicy, - ecPolicy); + ecPolicy, + loc); } - private static INodeAttributes getINodeAttributes( - FSDirectory fsd, String fullPath, byte[] path, INode node, int snapshot) { - return fsd.getAttributes(fullPath, path, node, snapshot); - } - - /** - * Create FileStatus with location info by file INode - * @param iip the INodesInPath containing the target INode and its ancestors - */ - private static HdfsFileStatus createLocatedFileStatus( - FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs, - byte storagePolicy, INodesInPath iip) throws IOException { - assert fsd.hasReadLock(); - long size = 0; // length is zero for directories - short replication = 0; - long blocksize = 0; - LocatedBlocks loc = null; - final boolean isEncrypted; - final INode node = iip.getLastINode(); - final int snapshot = iip.getPathSnapshotId(); - final boolean isRawPath = iip.isRaw(); - - final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp - .getFileEncryptionInfo(fsd, node, snapshot, iip); - final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy( - fsd.getFSNamesystem(), iip); - if (node.isFile()) { - final INodeFile fileNode = node.asFile(); - size = fileNode.computeFileSize(snapshot); - replication = fileNode.getFileReplication(snapshot); - blocksize = fileNode.getPreferredBlockSize(); - - final boolean inSnapshot = snapshot != Snapshot.CURRENT_STATE_ID; - final boolean isUc = !inSnapshot && fileNode.isUnderConstruction(); - final long fileSize = !inSnapshot && isUc ? - fileNode.computeFileSizeNotIncludingLastUcBlock() : size; - - loc = fsd.getBlockManager().createLocatedBlocks( - fileNode.getBlocks(snapshot), fileSize, isUc, 0L, size, false, - inSnapshot, feInfo, ecPolicy); - if (loc == null) { - loc = new LocatedBlocks(); - } - isEncrypted = (feInfo != null) - || (isRawPath && FSDirEncryptionZoneOp.isInAnEZ(fsd, iip)); + private static HdfsFileStatus createFileStatus(long length, boolean isdir, + int replication, long blocksize, long mtime, + long atime, FsPermission permission, String owner, String group, + byte[] symlink, byte[] path, long fileId, int childrenNum, + FileEncryptionInfo feInfo, byte storagePolicy, + ErasureCodingPolicy ecPolicy, LocatedBlocks locations) { + if (locations == null) { + return new HdfsFileStatus(length, isdir, replication, blocksize, + mtime, atime, permission, owner, group, symlink, path, fileId, + childrenNum, feInfo, storagePolicy, ecPolicy); } else { - isEncrypted = FSDirEncryptionZoneOp.isInAnEZ(fsd, iip); + return new HdfsLocatedFileStatus(length, isdir, replication, blocksize, + mtime, atime, permission, owner, group, symlink, path, fileId, + locations, childrenNum, feInfo, storagePolicy, ecPolicy); } - int childrenNum = node.isDirectory() ? - node.asDirectory().getChildrenNum(snapshot) : 0; - - HdfsLocatedFileStatus status = - new HdfsLocatedFileStatus(size, node.isDirectory(), replication, - blocksize, node.getModificationTime(snapshot), - node.getAccessTime(snapshot), - getPermissionForFileStatus(nodeAttrs, isEncrypted), - nodeAttrs.getUserName(), nodeAttrs.getGroupName(), - node.isSymlink() ? node.asSymlink().getSymlink() : null, path, - node.getId(), loc, childrenNum, feInfo, storagePolicy, ecPolicy); - // Set caching information for the located blocks. - if (loc != null) { - CacheManager cacheManager = fsd.getFSNamesystem().getCacheManager(); - for (LocatedBlock lb: loc.getLocatedBlocks()) { - cacheManager.setCachedLocations(lb); - } - } - return status; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java index 6938a848487..71362f8be6d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java @@ -58,7 +58,7 @@ class FSDirSymlinkOp { iip = fsd.resolvePathForWrite(pc, link, false); link = iip.getPath(); if (!createParent) { - fsd.verifyParentDir(iip, link); + fsd.verifyParentDir(iip); } if (!fsd.isValidToCreate(link, iip)) { throw new IOException( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index aa2be921d91..aab0f763e7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -19,14 +19,10 @@ package org.apache.hadoop.hdfs.server.namenode; import com.google.common.base.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; -import org.apache.hadoop.crypto.CipherSuite; -import org.apache.hadoop.crypto.CryptoProtocolVersion; -import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.hdfs.AddBlockFlag; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileEncryptionInfo; -import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.FsAction; @@ -38,7 +34,6 @@ import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.FSLimitException; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -307,6 +302,37 @@ class FSDirWriteFileOp { return clientNode; } + static INodesInPath resolvePathForStartFile(FSDirectory dir, + FSPermissionChecker pc, String src, EnumSet flag, + boolean createParent) throws IOException { + INodesInPath iip = dir.resolvePathForWrite(pc, src); + if (dir.isPermissionEnabled()) { + dir.checkAncestorAccess(pc, iip, FsAction.WRITE); + } + INode inode = iip.getLastINode(); + if (inode != null) { + // Verify that the destination does not exist as a directory already. + if (inode.isDirectory()) { + throw new FileAlreadyExistsException(iip.getPath() + + " already exists as a directory"); + } + // Verifies it's indeed a file and perms allow overwrite + INodeFile.valueOf(inode, src); + if (dir.isPermissionEnabled() && flag.contains(CreateFlag.OVERWRITE)) { + dir.checkPathAccess(pc, iip, FsAction.WRITE); + } + } else { + if (!createParent) { + dir.verifyParentDir(iip); + } + if (!flag.contains(CreateFlag.CREATE)) { + throw new FileNotFoundException("Can't overwrite non-existent " + src); + } + } + return iip; + } + + /** * Create a new file or overwrite an existing file
    * @@ -317,88 +343,21 @@ class FSDirWriteFileOp { * {@link ClientProtocol#create} */ static HdfsFileStatus startFile( - FSNamesystem fsn, FSPermissionChecker pc, String src, + FSNamesystem fsn, INodesInPath iip, PermissionStatus permissions, String holder, String clientMachine, EnumSet flag, boolean createParent, short replication, long blockSize, - EncryptionKeyInfo ezInfo, INode.BlocksMapUpdateInfo toRemoveBlocks, + FileEncryptionInfo feInfo, INode.BlocksMapUpdateInfo toRemoveBlocks, boolean logRetryEntry) throws IOException { assert fsn.hasWriteLock(); - - boolean create = flag.contains(CreateFlag.CREATE); boolean overwrite = flag.contains(CreateFlag.OVERWRITE); boolean isLazyPersist = flag.contains(CreateFlag.LAZY_PERSIST); - CipherSuite suite = null; - CryptoProtocolVersion version = null; - KeyProviderCryptoExtension.EncryptedKeyVersion edek = null; - - if (ezInfo != null) { - edek = ezInfo.edek; - suite = ezInfo.suite; - version = ezInfo.protocolVersion; - } - + final String src = iip.getPath(); FSDirectory fsd = fsn.getFSDirectory(); - INodesInPath iip = fsd.resolvePathForWrite(pc, src); - src = iip.getPath(); - // Verify that the destination does not exist as a directory already. - final INode inode = iip.getLastINode(); - if (inode != null && inode.isDirectory()) { - throw new FileAlreadyExistsException(src + - " already exists as a directory"); - } - - if (FSDirectory.isExactReservedName(src) || (FSDirectory.isReservedName(src) - && !FSDirectory.isReservedRawName(src) - && !FSDirectory.isReservedInodesName(src))) { - throw new InvalidPathException(src); - } - - final INodeFile myFile = INodeFile.valueOf(inode, src, true); - if (fsd.isPermissionEnabled()) { - if (overwrite && myFile != null) { - fsd.checkPathAccess(pc, iip, FsAction.WRITE); - } - /* - * To overwrite existing file, need to check 'w' permission - * of parent (equals to ancestor in this case) - */ - fsd.checkAncestorAccess(pc, iip, FsAction.WRITE); - } - - if (!createParent) { - fsd.verifyParentDir(iip, src); - } - - if (myFile == null && !create) { - throw new FileNotFoundException("Can't overwrite non-existent " + - src + " for client " + clientMachine); - } - - FileEncryptionInfo feInfo = null; - - final EncryptionZone zone = FSDirEncryptionZoneOp.getEZForPath(fsd, iip); - if (zone != null) { - // The path is now within an EZ, but we're missing encryption parameters - if (suite == null || edek == null) { - throw new RetryStartFileException(); - } - // Path is within an EZ and we have provided encryption parameters. - // Make sure that the generated EDEK matches the settings of the EZ. - final String ezKeyName = zone.getKeyName(); - if (!ezKeyName.equals(edek.getEncryptionKeyName())) { - throw new RetryStartFileException(); - } - feInfo = new FileEncryptionInfo(suite, version, - edek.getEncryptedKeyVersion().getMaterial(), - edek.getEncryptedKeyIv(), - ezKeyName, edek.getEncryptionKeyVersionName()); - } - - if (myFile != null) { + if (iip.getLastINode() != null) { if (overwrite) { List toRemoveINodes = new ChunkedArrayList<>(); List toRemoveUCFiles = new ChunkedArrayList<>(); @@ -433,11 +392,9 @@ class FSDirWriteFileOp { newNode.getFileUnderConstructionFeature().getClientName(), newNode.getId()); if (feInfo != null) { - FSDirEncryptionZoneOp.setFileEncryptionInfo(fsd, src, feInfo); - newNode = fsd.getInode(newNode.getId()).asFile(); + FSDirEncryptionZoneOp.setFileEncryptionInfo(fsd, iip, feInfo); } - setNewINodeStoragePolicy(fsd.getBlockManager(), newNode, iip, - isLazyPersist); + setNewINodeStoragePolicy(fsd.getBlockManager(), iip, isLazyPersist); fsd.getEditLog().logOpenFile(src, newNode, overwrite, logRetryEntry); if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: added " + @@ -446,30 +403,6 @@ class FSDirWriteFileOp { return FSDirStatAndListingOp.getFileInfo(fsd, iip); } - static EncryptionKeyInfo getEncryptionKeyInfo(FSNamesystem fsn, - FSPermissionChecker pc, String src, - CryptoProtocolVersion[] supportedVersions) - throws IOException { - FSDirectory fsd = fsn.getFSDirectory(); - INodesInPath iip = fsd.resolvePathForWrite(pc, src); - // Nothing to do if the path is not within an EZ - final EncryptionZone zone = FSDirEncryptionZoneOp.getEZForPath(fsd, iip); - if (zone == null) { - return null; - } - CryptoProtocolVersion protocolVersion = fsn.chooseProtocolVersion( - zone, supportedVersions); - CipherSuite suite = zone.getSuite(); - String ezKeyName = zone.getKeyName(); - - Preconditions.checkNotNull(protocolVersion); - Preconditions.checkNotNull(suite); - Preconditions.checkArgument(!suite.equals(CipherSuite.UNKNOWN), - "Chose an UNKNOWN CipherSuite!"); - Preconditions.checkNotNull(ezKeyName); - return new EncryptionKeyInfo(protocolVersion, suite, ezKeyName); - } - static INodeFile addFileForEditLog( FSDirectory fsd, long id, INodesInPath existing, byte[] localName, PermissionStatus permissions, List aclEntries, @@ -828,10 +761,9 @@ class FSDirWriteFileOp { NameNode.stateChangeLog.info(sb.toString()); } - private static void setNewINodeStoragePolicy(BlockManager bm, INodeFile - inode, INodesInPath iip, boolean isLazyPersist) - throws IOException { - + private static void setNewINodeStoragePolicy(BlockManager bm, + INodesInPath iip, boolean isLazyPersist) throws IOException { + INodeFile inode = iip.getLastINode().asFile(); if (isLazyPersist) { BlockStoragePolicy lpPolicy = bm.getStoragePolicy("LAZY_PERSIST"); @@ -887,19 +819,4 @@ class FSDirWriteFileOp { this.isStriped = isStriped; } } - - static class EncryptionKeyInfo { - final CryptoProtocolVersion protocolVersion; - final CipherSuite suite; - final String ezKeyName; - KeyProviderCryptoExtension.EncryptedKeyVersion edek; - - EncryptionKeyInfo( - CryptoProtocolVersion protocolVersion, CipherSuite suite, - String ezKeyName) { - this.protocolVersion = protocolVersion; - this.suite = suite; - this.ezKeyName = ezKeyName; - } - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java index 746fdb7c403..6badf2472fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java @@ -75,7 +75,7 @@ class FSDirXAttrOp { iip = fsd.resolvePathForWrite(pc, src); src = iip.getPath(); checkXAttrChangeAccess(fsd, iip, xAttr, pc); - unprotectedSetXAttrs(fsd, src, xAttrs, flag); + unprotectedSetXAttrs(fsd, iip, xAttrs, flag); } finally { fsd.writeUnlock(); } @@ -253,14 +253,11 @@ class FSDirXAttrOp { } static INode unprotectedSetXAttrs( - FSDirectory fsd, final String src, final List xAttrs, + FSDirectory fsd, final INodesInPath iip, final List xAttrs, final EnumSet flag) throws IOException { assert fsd.hasWriteLock(); - INodesInPath iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), - true); INode inode = FSDirectory.resolveLastINode(iip); - int snapshotId = iip.getLatestSnapshotId(); List existingXAttrs = XAttrStorage.readINodeXAttrs(inode); List newXAttrs = setINodeXAttrs(fsd, existingXAttrs, xAttrs, flag); final boolean isFile = inode.isFile(); @@ -287,7 +284,7 @@ class FSDirXAttrOp { } } - XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId); + XAttrStorage.updateINodeXAttrs(inode, newXAttrs, iip.getLatestSnapshotId()); return inode; } @@ -361,22 +358,20 @@ class FSDirXAttrOp { return xAttrs; } - static XAttr getXAttrByPrefixedName(FSDirectory fsd, INode inode, - int snapshotId, String prefixedName) throws IOException { + static XAttr getXAttrByPrefixedName(FSDirectory fsd, INodesInPath iip, + String prefixedName) throws IOException { fsd.readLock(); try { - return XAttrStorage.readINodeXAttrByPrefixedName(inode, snapshotId, - prefixedName); + return XAttrStorage.readINodeXAttrByPrefixedName(iip, prefixedName); } finally { fsd.readUnlock(); } } static XAttr unprotectedGetXAttrByPrefixedName( - INode inode, int snapshotId, String prefixedName) + INodesInPath iip, String prefixedName) throws IOException { - return XAttrStorage.readINodeXAttrByPrefixedName(inode, snapshotId, - prefixedName); + return XAttrStorage.readINodeXAttrByPrefixedName(iip, prefixedName); } private static void checkXAttrChangeAccess( @@ -429,11 +424,7 @@ class FSDirXAttrOp { throws IOException { fsd.readLock(); try { - String src = iip.getPath(); - INode inode = FSDirectory.resolveLastINode(iip); - int snapshotId = iip.getPathSnapshotId(); - return XAttrStorage.readINodeXAttrs(fsd.getAttributes(src, - inode.getLocalNameBytes(), inode, snapshotId)); + return XAttrStorage.readINodeXAttrs(fsd.getAttributes(iip)); } finally { fsd.readUnlock(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 2c7a2680efb..a059ee56bfb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -1744,11 +1744,10 @@ public class FSDirectory implements Closeable { } } - void checkUnreadableBySuperuser( - FSPermissionChecker pc, INode inode, int snapshotId) + void checkUnreadableBySuperuser(FSPermissionChecker pc, INodesInPath iip) throws IOException { if (pc.isSuperUser()) { - if (FSDirXAttrOp.getXAttrByPrefixedName(this, inode, snapshotId, + if (FSDirXAttrOp.getXAttrByPrefixedName(this, iip, SECURITY_XATTR_UNREADABLE_BY_SUPERUSER) != null) { throw new AccessControlException( "Access is denied for " + pc.getUser() + " since the superuser " @@ -1766,17 +1765,16 @@ public class FSDirectory implements Closeable { /** * Verify that parent directory of src exists. */ - void verifyParentDir(INodesInPath iip, String src) + void verifyParentDir(INodesInPath iip) throws FileNotFoundException, ParentNotDirectoryException { - Path parent = new Path(src).getParent(); - if (parent != null) { + if (iip.length() > 2) { final INode parentNode = iip.getINode(-2); if (parentNode == null) { throw new FileNotFoundException("Parent directory doesn't exist: " - + parent); - } else if (!parentNode.isDirectory() && !parentNode.isSymlink()) { + + iip.getParentPath()); + } else if (!parentNode.isDirectory()) { throw new ParentNotDirectoryException("Parent path is not a directory: " - + parent); + + iip.getParentPath()); } } } @@ -1807,14 +1805,19 @@ public class FSDirectory implements Closeable { inodeId.setCurrentValue(newValue); } - INodeAttributes getAttributes(String fullPath, byte[] path, - INode node, int snapshot) { + INodeAttributes getAttributes(INodesInPath iip) + throws FileNotFoundException { + INode node = FSDirectory.resolveLastINode(iip); + int snapshot = iip.getPathSnapshotId(); INodeAttributes nodeAttrs = node.getSnapshotINode(snapshot); if (attributeProvider != null) { - fullPath = fullPath - + (fullPath.endsWith(Path.SEPARATOR) ? "" : Path.SEPARATOR) - + DFSUtil.bytes2String(path); - nodeAttrs = attributeProvider.getAttributes(fullPath, nodeAttrs); + // permission checking sends the full components array including the + // first empty component for the root. however file status + // related calls are expected to strip out the root component according + // to TestINodeAttributeProvider. + byte[][] components = iip.getPathComponents(); + components = Arrays.copyOfRange(components, 1, components.length); + nodeAttrs = attributeProvider.getAttributes(components, nodeAttrs); } return nodeAttrs; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index 25f5a4f5c4e..8abdba85736 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -356,7 +356,7 @@ public class FSEditLogLoader { INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path, true); if (oldFile != null && addCloseOp.overwrite) { // This is OP_ADD with overwrite - FSDirDeleteOp.deleteForEditLog(fsDir, path, addCloseOp.mtime); + FSDirDeleteOp.deleteForEditLog(fsDir, iip, addCloseOp.mtime); iip = INodesInPath.replace(iip, iip.length() - 1, null); oldFile = null; } @@ -383,10 +383,8 @@ public class FSEditLogLoader { // add the op into retry cache if necessary if (toAddRetryCache) { - HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog( - fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, - HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID, - false, iip); + HdfsFileStatus stat = + FSDirStatAndListingOp.createFileStatusForEditLog(fsDir, iip); fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId, addCloseOp.rpcCallId, stat); } @@ -402,10 +400,8 @@ public class FSEditLogLoader { false); // add the op into retry cache if necessary if (toAddRetryCache) { - HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog( - fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, - HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, - Snapshot.CURRENT_STATE_ID, false, iip); + HdfsFileStatus stat = + FSDirStatAndListingOp.createFileStatusForEditLog(fsDir, iip); fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId, addCloseOp.rpcCallId, new LastBlockWithStatus(lb, stat)); } @@ -480,10 +476,8 @@ public class FSEditLogLoader { false, false); // add the op into retry cache if necessary if (toAddRetryCache) { - HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog( - fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, - HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, - Snapshot.CURRENT_STATE_ID, false, iip); + HdfsFileStatus stat = + FSDirStatAndListingOp.createFileStatusForEditLog(fsDir, iip); fsNamesys.addCacheEntryWithPayload(appendOp.rpcClientId, appendOp.rpcCallId, new LastBlockWithStatus(lb, stat)); } @@ -527,10 +521,12 @@ public class FSEditLogLoader { } case OP_SET_REPLICATION: { SetReplicationOp setReplicationOp = (SetReplicationOp)op; + String src = renameReservedPathsOnUpgrade( + setReplicationOp.path, logVersion); + INodesInPath iip = fsDir.getINodesInPath4Write(src); short replication = fsNamesys.getBlockManager().adjustReplication( setReplicationOp.replication); - FSDirAttrOp.unprotectedSetReplication(fsDir, renameReservedPathsOnUpgrade( - setReplicationOp.path, logVersion), replication); + FSDirAttrOp.unprotectedSetReplication(fsDir, iip, replication); break; } case OP_CONCAT_DELETE: { @@ -569,10 +565,11 @@ public class FSEditLogLoader { } case OP_DELETE: { DeleteOp deleteOp = (DeleteOp)op; - FSDirDeleteOp.deleteForEditLog( - fsDir, renameReservedPathsOnUpgrade(deleteOp.path, logVersion), - deleteOp.timestamp); - + final String src = renameReservedPathsOnUpgrade( + deleteOp.path, logVersion); + final INodesInPath iip = fsDir.getINodesInPath4Write(src, false); + FSDirDeleteOp.deleteForEditLog(fsDir, iip, deleteOp.timestamp); + if (toAddRetryCache) { fsNamesys.addCacheEntry(deleteOp.rpcClientId, deleteOp.rpcCallId); } @@ -595,52 +592,66 @@ public class FSEditLogLoader { } case OP_SET_PERMISSIONS: { SetPermissionsOp setPermissionsOp = (SetPermissionsOp)op; - FSDirAttrOp.unprotectedSetPermission(fsDir, renameReservedPathsOnUpgrade( - setPermissionsOp.src, logVersion), setPermissionsOp.permissions); + final String src = + renameReservedPathsOnUpgrade(setPermissionsOp.src, logVersion); + final INodesInPath iip = fsDir.getINodesInPath4Write(src); + FSDirAttrOp.unprotectedSetPermission(fsDir, iip, + setPermissionsOp.permissions); break; } case OP_SET_OWNER: { SetOwnerOp setOwnerOp = (SetOwnerOp)op; - FSDirAttrOp.unprotectedSetOwner( - fsDir, renameReservedPathsOnUpgrade(setOwnerOp.src, logVersion), + final String src = renameReservedPathsOnUpgrade( + setOwnerOp.src, logVersion); + final INodesInPath iip = fsDir.getINodesInPath4Write(src); + FSDirAttrOp.unprotectedSetOwner(fsDir, iip, setOwnerOp.username, setOwnerOp.groupname); break; } case OP_SET_NS_QUOTA: { SetNSQuotaOp setNSQuotaOp = (SetNSQuotaOp)op; - FSDirAttrOp.unprotectedSetQuota( - fsDir, renameReservedPathsOnUpgrade(setNSQuotaOp.src, logVersion), + final String src = renameReservedPathsOnUpgrade( + setNSQuotaOp.src, logVersion); + final INodesInPath iip = fsDir.getINodesInPath4Write(src); + FSDirAttrOp.unprotectedSetQuota(fsDir, iip, setNSQuotaOp.nsQuota, HdfsConstants.QUOTA_DONT_SET, null); break; } case OP_CLEAR_NS_QUOTA: { ClearNSQuotaOp clearNSQuotaOp = (ClearNSQuotaOp)op; - FSDirAttrOp.unprotectedSetQuota( - fsDir, renameReservedPathsOnUpgrade(clearNSQuotaOp.src, logVersion), + final String src = renameReservedPathsOnUpgrade( + clearNSQuotaOp.src, logVersion); + final INodesInPath iip = fsDir.getINodesInPath4Write(src); + FSDirAttrOp.unprotectedSetQuota(fsDir, iip, HdfsConstants.QUOTA_RESET, HdfsConstants.QUOTA_DONT_SET, null); break; } - - case OP_SET_QUOTA: + case OP_SET_QUOTA: { SetQuotaOp setQuotaOp = (SetQuotaOp) op; - FSDirAttrOp.unprotectedSetQuota(fsDir, - renameReservedPathsOnUpgrade(setQuotaOp.src, logVersion), + final String src = renameReservedPathsOnUpgrade( + setQuotaOp.src, logVersion); + final INodesInPath iip = fsDir.getINodesInPath4Write(src); + FSDirAttrOp.unprotectedSetQuota(fsDir, iip, setQuotaOp.nsQuota, setQuotaOp.dsQuota, null); break; - - case OP_SET_QUOTA_BY_STORAGETYPE: - FSEditLogOp.SetQuotaByStorageTypeOp setQuotaByStorageTypeOp = + } + case OP_SET_QUOTA_BY_STORAGETYPE: { + FSEditLogOp.SetQuotaByStorageTypeOp setQuotaByStorageTypeOp = (FSEditLogOp.SetQuotaByStorageTypeOp) op; - FSDirAttrOp.unprotectedSetQuota(fsDir, - renameReservedPathsOnUpgrade(setQuotaByStorageTypeOp.src, logVersion), + final String src = renameReservedPathsOnUpgrade( + setQuotaByStorageTypeOp.src, logVersion); + final INodesInPath iip = fsDir.getINodesInPath4Write(src); + FSDirAttrOp.unprotectedSetQuota(fsDir, iip, HdfsConstants.QUOTA_DONT_SET, setQuotaByStorageTypeOp.dsQuota, setQuotaByStorageTypeOp.type); - break; - + break; + } case OP_TIMES: { TimesOp timesOp = (TimesOp)op; - FSDirAttrOp.unprotectedSetTimes( - fsDir, renameReservedPathsOnUpgrade(timesOp.path, logVersion), + final String src = renameReservedPathsOnUpgrade( + timesOp.path, logVersion); + final INodesInPath iip = fsDir.getINodesInPath4Write(src); + FSDirAttrOp.unprotectedSetTimes(fsDir, iip, timesOp.mtime, timesOp.atime, true); break; } @@ -902,7 +913,8 @@ public class FSEditLogLoader { } case OP_SET_XATTR: { SetXAttrOp setXAttrOp = (SetXAttrOp) op; - FSDirXAttrOp.unprotectedSetXAttrs(fsDir, setXAttrOp.src, + INodesInPath iip = fsDir.getINodesInPath4Write(setXAttrOp.src); + FSDirXAttrOp.unprotectedSetXAttrs(fsDir, iip, setXAttrOp.xAttrs, EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 47002630a05..8c591861fca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -71,12 +71,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_EXPIRYTIME_MILLIS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_EXPIRYTIME_MILLIS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_HEAP_PERCENT_DEFAULT; @@ -95,6 +89,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*; import static org.apache.hadoop.util.Time.now; import static org.apache.hadoop.util.Time.monotonicNow; +import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; @@ -129,8 +124,6 @@ import java.util.TreeMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -153,6 +146,7 @@ import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsServerDefaults; @@ -224,6 +218,7 @@ import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.Util; +import org.apache.hadoop.hdfs.server.namenode.FSDirEncryptionZoneOp.EncryptionKeyInfo; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream; @@ -247,7 +242,6 @@ import org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger; import org.apache.hadoop.hdfs.server.namenode.top.TopConf; import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics; import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager; -import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; @@ -284,7 +278,6 @@ import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Timer; import org.apache.hadoop.util.VersionInfo; import org.apache.log4j.Appender; import org.apache.log4j.AsyncAppender; @@ -344,7 +337,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, private void logAuditEvent(boolean succeeded, String cmd, String src, String dst, HdfsFileStatus stat) throws IOException { if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(succeeded, getRemoteUser(), getRemoteIp(), + logAuditEvent(succeeded, Server.getRemoteUser(), Server.getRemoteIp(), cmd, src, dst, stat); } } @@ -713,12 +706,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, LOG.info("Enabling async auditlog"); enableAsyncAuditLog(); } - boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true); - LOG.info("fsLock is fair:" + fair); - fsLock = new FSNamesystemLock(fair); - cond = fsLock.writeLock().newCondition(); + fsLock = new FSNamesystemLock(conf); + cond = fsLock.newWriteLockCondition(); cpLock = new ReentrantLock(); - setTimer(new Timer()); this.fsImage = fsImage; try { @@ -827,17 +817,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_KEY, DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT); - this.writeLockReportingThreshold = conf.getLong( - DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY, - DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT); - this.readLockReportingThreshold = conf.getLong( - DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY, - DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT); - - this.lockSuppressWarningInterval = conf.getTimeDuration( - DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY, - DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); - // For testing purposes, allow the DT secret manager to be started regardless // of whether security is enabled. alwaysUseDelegationTokensForTests = conf.getBoolean( @@ -1010,6 +989,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, // Add audit logger to calculate top users if (topConf.isEnabled) { topMetrics = new TopMetrics(conf, topConf.nntopReportingPeriodsMs); + if (DefaultMetricsSystem.instance().getSource( + TOPMETRICS_METRICS_SOURCE_NAME) == null) { + DefaultMetricsSystem.instance().register(TOPMETRICS_METRICS_SOURCE_NAME, + "Top N operations by user", topMetrics); + } auditLoggers.add(new TopAuditLogger(topMetrics)); } @@ -1516,131 +1500,25 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, return Util.stringCollectionAsURIs(dirNames); } - private final long lockSuppressWarningInterval; - /** Threshold (ms) for long holding write lock report. */ - private final long writeLockReportingThreshold; - private int numWriteLockWarningsSuppressed = 0; - private long timeStampOfLastWriteLockReport = 0; - private long longestWriteLockHeldInterval = 0; - /** Last time stamp for write lock. Keep the longest one for multi-entrance.*/ - private long writeLockHeldTimeStamp; - /** Threshold (ms) for long holding read lock report. */ - private long readLockReportingThreshold; - private AtomicInteger numReadLockWarningsSuppressed = new AtomicInteger(0); - private AtomicLong timeStampOfLastReadLockReport = new AtomicLong(0); - private AtomicLong longestReadLockHeldInterval = new AtomicLong(0); - private Timer timer; - /** - * Last time stamp for read lock. Keep the longest one for - * multi-entrance. This is ThreadLocal since there could be - * many read locks held simultaneously. - */ - private static ThreadLocal readLockHeldTimeStamp = - new ThreadLocal() { - @Override - public Long initialValue() { - return Long.MAX_VALUE; - } - }; - @Override public void readLock() { - this.fsLock.readLock().lock(); - if (this.fsLock.getReadHoldCount() == 1) { - readLockHeldTimeStamp.set(timer.monotonicNow()); - } + this.fsLock.readLock(); } @Override public void readUnlock() { - final boolean needReport = this.fsLock.getReadHoldCount() == 1; - final long readLockInterval = timer.monotonicNow() - - readLockHeldTimeStamp.get(); - if (needReport) { - readLockHeldTimeStamp.remove(); - } - - this.fsLock.readLock().unlock(); - - if (needReport && readLockInterval >= this.readLockReportingThreshold) { - long localLongestReadLock; - do { - localLongestReadLock = longestReadLockHeldInterval.get(); - } while (localLongestReadLock - readLockInterval < 0 - && !longestReadLockHeldInterval.compareAndSet(localLongestReadLock, - readLockInterval)); - - long localTimeStampOfLastReadLockReport; - long now; - do { - now = timer.monotonicNow(); - localTimeStampOfLastReadLockReport = timeStampOfLastReadLockReport - .get(); - if (now - localTimeStampOfLastReadLockReport < - lockSuppressWarningInterval) { - numReadLockWarningsSuppressed.incrementAndGet(); - return; - } - } while (!timeStampOfLastReadLockReport.compareAndSet( - localTimeStampOfLastReadLockReport, now)); - int numSuppressedWarnings = numReadLockWarningsSuppressed.getAndSet(0); - long longestLockHeldInterval = longestReadLockHeldInterval.getAndSet(0); - LOG.info("FSNamesystem read lock held for " + readLockInterval + - " ms via\n" + StringUtils.getStackTrace(Thread.currentThread()) + - "\tNumber of suppressed read-lock reports: " + - numSuppressedWarnings + "\n\tLongest read-lock held interval: " + - longestLockHeldInterval); - } + this.fsLock.readUnlock(); } @Override public void writeLock() { - this.fsLock.writeLock().lock(); - if (fsLock.getWriteHoldCount() == 1) { - writeLockHeldTimeStamp = timer.monotonicNow(); - } + this.fsLock.writeLock(); } @Override public void writeLockInterruptibly() throws InterruptedException { - this.fsLock.writeLock().lockInterruptibly(); - if (fsLock.getWriteHoldCount() == 1) { - writeLockHeldTimeStamp = timer.monotonicNow(); - } + this.fsLock.writeLockInterruptibly(); } @Override public void writeUnlock() { - final boolean needReport = fsLock.getWriteHoldCount() == 1 && - fsLock.isWriteLockedByCurrentThread(); - final long currentTime = timer.monotonicNow(); - final long writeLockInterval = currentTime - writeLockHeldTimeStamp; - - boolean logReport = false; - int numSuppressedWarnings = 0; - long longestLockHeldInterval = 0; - if (needReport && writeLockInterval >= this.writeLockReportingThreshold) { - if (writeLockInterval > longestWriteLockHeldInterval) { - longestWriteLockHeldInterval = writeLockInterval; - } - if (currentTime - timeStampOfLastWriteLockReport > this - .lockSuppressWarningInterval) { - logReport = true; - numSuppressedWarnings = numWriteLockWarningsSuppressed; - numWriteLockWarningsSuppressed = 0; - longestLockHeldInterval = longestWriteLockHeldInterval; - longestWriteLockHeldInterval = 0; - timeStampOfLastWriteLockReport = currentTime; - } else { - numWriteLockWarningsSuppressed++; - } - } - - this.fsLock.writeLock().unlock(); - - if (logReport) { - LOG.info("FSNamesystem write lock held for " + writeLockInterval + - " ms via\n" + StringUtils.getStackTrace(Thread.currentThread()) + - "\tNumber of suppressed write-lock reports: " + - numSuppressedWarnings + "\n\tLongest write-lock held interval: " + - longestLockHeldInterval); - } + this.fsLock.writeUnlock(); } @Override public boolean hasWriteLock() { @@ -1925,8 +1803,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, boolean updateAccessTime = inode != null && now > inode.getAccessTime() + dir.getAccessTimePrecision(); if (!isInSafeMode() && updateAccessTime) { - boolean changed = FSDirAttrOp.setTimes(dir, - inode, -1, now, false, iip.getLatestSnapshotId()); + boolean changed = FSDirAttrOp.setTimes(dir, iip, -1, now, false); if (changed) { getEditLog().logTimes(src, -1, now); } @@ -2281,7 +2158,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, return status; } - private HdfsFileStatus startFileInt(final String src, + private HdfsFileStatus startFileInt(String src, PermissionStatus permissions, String holder, String clientMachine, EnumSet flag, boolean createParent, short replication, long blockSize, CryptoProtocolVersion[] supportedVersions, @@ -2300,92 +2177,79 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, .append(Arrays.toString(supportedVersions)); NameNode.stateChangeLog.debug(builder.toString()); } - if (!DFSUtil.isValidName(src)) { + if (!DFSUtil.isValidName(src) || + FSDirectory.isExactReservedName(src) || + (FSDirectory.isReservedName(src) + && !FSDirectory.isReservedRawName(src) + && !FSDirectory.isReservedInodesName(src))) { throw new InvalidPathException(src); } - checkOperation(OperationCategory.READ); - readLock(); - try { - checkOperation(OperationCategory.READ); - if (!FSDirErasureCodingOp.hasErasureCodingPolicy(this, src)) { - blockManager.verifyReplication(src, replication, clientMachine); - } - } finally { - readUnlock(); - } - - checkOperation(OperationCategory.WRITE); - if (blockSize < minBlockSize) { - throw new IOException("Specified block size is less than configured" + - " minimum value (" + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY - + "): " + blockSize + " < " + minBlockSize); - } - FSPermissionChecker pc = getPermissionChecker(); - - /** - * If the file is in an encryption zone, we optimistically create an - * EDEK for the file by calling out to the configured KeyProvider. - * Since this typically involves doing an RPC, we take the readLock - * initially, then drop it to do the RPC. - * - * Since the path can flip-flop between being in an encryption zone and not - * in the meantime, we need to recheck the preconditions when we retake the - * lock to do the create. If the preconditions are not met, we throw a - * special RetryStartFileException to ask the DFSClient to try the create - * again later. - */ - FSDirWriteFileOp.EncryptionKeyInfo ezInfo = null; - - if (provider != null) { - readLock(); - try { - checkOperation(OperationCategory.READ); - ezInfo = FSDirWriteFileOp - .getEncryptionKeyInfo(this, pc, src, supportedVersions); - } finally { - readUnlock(); - } - - // Generate EDEK if necessary while not holding the lock - if (ezInfo != null) { - ezInfo.edek = FSDirEncryptionZoneOp - .generateEncryptedDataEncryptionKey(dir, ezInfo.ezKeyName); - } - EncryptionFaultInjector.getInstance().startFileAfterGenerateKey(); - } - - boolean skipSync = false; + INodesInPath iip = null; + boolean skipSync = true; // until we do something that might create edits HdfsFileStatus stat = null; + BlocksMapUpdateInfo toRemoveBlocks = null; - // Proceed with the create, using the computed cipher suite and - // generated EDEK - BlocksMapUpdateInfo toRemoveBlocks = new BlocksMapUpdateInfo(); + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot create file" + src); + + iip = FSDirWriteFileOp.resolvePathForStartFile( + dir, pc, src, flag, createParent); + + if (!FSDirErasureCodingOp.hasErasureCodingPolicy(this, iip)) { + blockManager.verifyReplication(src, replication, clientMachine); + } + + if (blockSize < minBlockSize) { + throw new IOException("Specified block size is less than configured" + + " minimum value (" + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY + + "): " + blockSize + " < " + minBlockSize); + } + + FileEncryptionInfo feInfo = null; + if (provider != null) { + EncryptionKeyInfo ezInfo = FSDirEncryptionZoneOp.getEncryptionKeyInfo( + this, iip, supportedVersions); + // if the path has an encryption zone, the lock was released while + // generating the EDEK. re-resolve the path to ensure the namesystem + // and/or EZ has not mutated + if (ezInfo != null) { + checkOperation(OperationCategory.WRITE); + iip = FSDirWriteFileOp.resolvePathForStartFile( + dir, pc, iip.getPath(), flag, createParent); + feInfo = FSDirEncryptionZoneOp.getFileEncryptionInfo( + dir, iip, ezInfo); + } + } + + skipSync = false; // following might generate edits + toRemoveBlocks = new BlocksMapUpdateInfo(); dir.writeLock(); try { - stat = FSDirWriteFileOp.startFile(this, pc, src, permissions, holder, + stat = FSDirWriteFileOp.startFile(this, iip, permissions, holder, clientMachine, flag, createParent, - replication, blockSize, ezInfo, + replication, blockSize, feInfo, toRemoveBlocks, logRetryCache); + } catch (IOException e) { + skipSync = e instanceof StandbyException; + throw e; } finally { dir.writeUnlock(); } - } catch (IOException e) { - skipSync = e instanceof StandbyException; - throw e; } finally { writeUnlock(); // There might be transactions logged while trying to recover the lease. // They need to be sync'ed even when an exception was thrown. if (!skipSync) { getEditLog().logSync(); - removeBlocks(toRemoveBlocks); - toRemoveBlocks.clear(); + if (toRemoveBlocks != null) { + removeBlocks(toRemoveBlocks); + toRemoveBlocks.clear(); + } } } @@ -3946,7 +3810,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, LOG.warn("Removing lazyPersist file " + bc.getName() + " with no replicas."); BlocksMapUpdateInfo toRemoveBlocks = FSDirDeleteOp.deleteInternal( - FSNamesystem.this, bc.getName(), + FSNamesystem.this, INodesInPath.fromINode((INodeFile) bc), false); changed |= toRemoveBlocks != null; if (toRemoveBlocks != null) { @@ -5397,17 +5261,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, * RPC call context even if the client exits. */ boolean isExternalInvocation() { - return Server.isRpcInvocation() || NamenodeWebHdfsMethods.isWebHdfsInvocation(); + return Server.isRpcInvocation(); } - private static InetAddress getRemoteIp() { - InetAddress ip = Server.getRemoteIp(); - if (ip != null) { - return ip; - } - return NamenodeWebHdfsMethods.getRemoteIp(); - } - // optimize ugi lookup for RPC operations to avoid a trip through // UGI.getCurrentUser which is synch'ed private static UserGroupInformation getRemoteUser() throws IOException { @@ -5554,7 +5410,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, VolumeFailureSummary volumeFailureSummary = node.getVolumeFailureSummary(); if (volumeFailureSummary != null) { innerinfo - .put("failedStorageLocations", + .put("failedStorageIDs", volumeFailureSummary.getFailedStorageLocations()) .put("lastVolumeFailureDate", volumeFailureSummary.getLastVolumeFailureDate()) @@ -7053,7 +6909,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, sb.append(trackingId); } sb.append("\t").append("proto="); - sb.append(NamenodeWebHdfsMethods.isWebHdfsInvocation() ? "webhdfs" : "rpc"); + sb.append(Server.getProtocol()); if (isCallerContextEnabled && callerContext != null && callerContext.isContextValid()) { @@ -7173,9 +7029,5 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, .size(); } - @VisibleForTesting - void setTimer(Timer newTimer) { - this.timer = newTimer; - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java index d2397961257..043f5693829 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java @@ -19,33 +19,186 @@ package org.apache.hadoop.hdfs.server.namenode; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantReadWriteLock; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Timer; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY; /** - * Mimics a ReentrantReadWriteLock so more sophisticated locking capabilities - * are possible. + * Mimics a ReentrantReadWriteLock but does not directly implement the interface + * so more sophisticated locking capabilities and logging/metrics are possible. */ -class FSNamesystemLock implements ReadWriteLock { +class FSNamesystemLock { @VisibleForTesting protected ReentrantReadWriteLock coarseLock; - - FSNamesystemLock(boolean fair) { + + private final Timer timer; + + /** + * Log statements about long lock hold times will not be produced more + * frequently than this interval. + */ + private final long lockSuppressWarningInterval; + + /** Threshold (ms) for long holding write lock report. */ + private final long writeLockReportingThreshold; + /** Last time stamp for write lock. Keep the longest one for multi-entrance.*/ + private long writeLockHeldTimeStamp; + private int numWriteLockWarningsSuppressed = 0; + private long timeStampOfLastWriteLockReport = 0; + private long longestWriteLockHeldInterval = 0; + + /** Threshold (ms) for long holding read lock report. */ + private final long readLockReportingThreshold; + /** + * Last time stamp for read lock. Keep the longest one for + * multi-entrance. This is ThreadLocal since there could be + * many read locks held simultaneously. + */ + private final ThreadLocal readLockHeldTimeStamp = + new ThreadLocal() { + @Override + public Long initialValue() { + return Long.MAX_VALUE; + } + }; + private final AtomicInteger numReadLockWarningsSuppressed = + new AtomicInteger(0); + private final AtomicLong timeStampOfLastReadLockReport = new AtomicLong(0); + private final AtomicLong longestReadLockHeldInterval = new AtomicLong(0); + + FSNamesystemLock(Configuration conf) { + this(conf, new Timer()); + } + + @VisibleForTesting + FSNamesystemLock(Configuration conf, Timer timer) { + boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true); + FSNamesystem.LOG.info("fsLock is fair: " + fair); this.coarseLock = new ReentrantReadWriteLock(fair); + this.timer = timer; + + this.writeLockReportingThreshold = conf.getLong( + DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY, + DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT); + this.readLockReportingThreshold = conf.getLong( + DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY, + DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT); + this.lockSuppressWarningInterval = conf.getTimeDuration( + DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY, + DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + } + + public void readLock() { + coarseLock.readLock().lock(); + if (coarseLock.getReadHoldCount() == 1) { + readLockHeldTimeStamp.set(timer.monotonicNow()); + } + } + + public void readUnlock() { + final boolean needReport = coarseLock.getReadHoldCount() == 1; + final long readLockInterval = + timer.monotonicNow() - readLockHeldTimeStamp.get(); + coarseLock.readLock().unlock(); + + if (needReport) { + readLockHeldTimeStamp.remove(); + } + if (needReport && readLockInterval >= this.readLockReportingThreshold) { + long localLongestReadLock; + do { + localLongestReadLock = longestReadLockHeldInterval.get(); + } while (localLongestReadLock - readLockInterval < 0 && + !longestReadLockHeldInterval.compareAndSet(localLongestReadLock, + readLockInterval)); + + long localTimeStampOfLastReadLockReport; + long now; + do { + now = timer.monotonicNow(); + localTimeStampOfLastReadLockReport = + timeStampOfLastReadLockReport.get(); + if (now - localTimeStampOfLastReadLockReport < + lockSuppressWarningInterval) { + numReadLockWarningsSuppressed.incrementAndGet(); + return; + } + } while (!timeStampOfLastReadLockReport.compareAndSet( + localTimeStampOfLastReadLockReport, now)); + int numSuppressedWarnings = numReadLockWarningsSuppressed.getAndSet(0); + long longestLockHeldInterval = longestReadLockHeldInterval.getAndSet(0); + FSNamesystem.LOG.info("FSNamesystem read lock held for " + + readLockInterval + " ms via\n" + + StringUtils.getStackTrace(Thread.currentThread()) + + "\tNumber of suppressed read-lock reports: " + numSuppressedWarnings + + "\n\tLongest read-lock held interval: " + longestLockHeldInterval); + } } - @Override - public Lock readLock() { - return coarseLock.readLock(); + public void writeLock() { + coarseLock.writeLock().lock(); + if (coarseLock.getWriteHoldCount() == 1) { + writeLockHeldTimeStamp = timer.monotonicNow(); + } } - - @Override - public Lock writeLock() { - return coarseLock.writeLock(); + + public void writeLockInterruptibly() throws InterruptedException { + coarseLock.writeLock().lockInterruptibly(); + if (coarseLock.getWriteHoldCount() == 1) { + writeLockHeldTimeStamp = timer.monotonicNow(); + } + } + + public void writeUnlock() { + final boolean needReport = coarseLock.getWriteHoldCount() == 1 && + coarseLock.isWriteLockedByCurrentThread(); + final long currentTime = timer.monotonicNow(); + final long writeLockInterval = currentTime - writeLockHeldTimeStamp; + + boolean logReport = false; + int numSuppressedWarnings = 0; + long longestLockHeldInterval = 0; + if (needReport && writeLockInterval >= this.writeLockReportingThreshold) { + if (writeLockInterval > longestWriteLockHeldInterval) { + longestWriteLockHeldInterval = writeLockInterval; + } + if (currentTime - timeStampOfLastWriteLockReport > + this.lockSuppressWarningInterval) { + logReport = true; + numSuppressedWarnings = numWriteLockWarningsSuppressed; + numWriteLockWarningsSuppressed = 0; + longestLockHeldInterval = longestWriteLockHeldInterval; + longestWriteLockHeldInterval = 0; + timeStampOfLastWriteLockReport = currentTime; + } else { + numWriteLockWarningsSuppressed++; + } + } + + coarseLock.writeLock().unlock(); + + if (logReport) { + FSNamesystem.LOG.info("FSNamesystem write lock held for " + + writeLockInterval + " ms via\n" + + StringUtils.getStackTrace(Thread.currentThread()) + + "\tNumber of suppressed write-lock reports: " + + numSuppressedWarnings + "\n\tLongest write-lock held interval: " + + longestLockHeldInterval); + } } public int getReadHoldCount() { @@ -60,6 +213,10 @@ class FSNamesystemLock implements ReadWriteLock { return coarseLock.isWriteLockedByCurrentThread(); } + public Condition newWriteLockCondition() { + return coarseLock.writeLock().newCondition(); + } + /** * Returns the QueueLength of waiting threads. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index c6258a173b3..e1db990c1f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -429,6 +429,7 @@ public abstract class INode implements INodeAttributes, Diff.Element { public final ContentSummary computeAndConvertContentSummary(int snapshotId, ContentSummaryComputationContext summary) { computeContentSummary(snapshotId, summary); + summary.tallyDeletedSnapshottedINodes(); final ContentCounts counts = summary.getCounts(); final ContentCounts snapshotCounts = summary.getSnapshotCounts(); final QuotaCounts q = getQuotaCounts(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java index 2e0775b3963..2f9bc370daf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributeProvider.java @@ -17,13 +17,6 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -import com.google.common.annotations.VisibleForTesting; - import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -87,7 +80,7 @@ public abstract class INodeAttributeProvider { */ public abstract void stop(); - @VisibleForTesting + @Deprecated String[] getPathElements(String path) { path = path.trim(); if (path.charAt(0) != Path.SEPARATOR_CHAR) { @@ -115,6 +108,7 @@ public abstract class INodeAttributeProvider { return pathElements; } + @Deprecated public INodeAttributes getAttributes(String fullPath, INodeAttributes inode) { return getAttributes(getPathElements(fullPath), inode); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index 24c881556db..b6e27135fea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -628,17 +628,10 @@ public class INodeDirectory extends INodeWithAdditionalFields @Override public ContentSummaryComputationContext computeContentSummary(int snapshotId, ContentSummaryComputationContext summary) { + summary.nodeIncluded(this); final DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature(); if (sf != null && snapshotId == Snapshot.CURRENT_STATE_ID) { - final ContentCounts counts = new ContentCounts.Builder().build(); - // if the getContentSummary call is against a non-snapshot path, the - // computation should include all the deleted files/directories - sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(), - counts); - summary.getCounts().addContents(counts); - // Also add ContentSummary to snapshotCounts (So we can extract it - // later from the ContentSummary of all). - summary.getSnapshotCounts().addContents(counts); + sf.computeContentSummary4Snapshot(summary); } final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature(); if (q != null && snapshotId == Snapshot.CURRENT_STATE_ID) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 12ead7fa3a0..37f97dbd7ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -770,6 +770,7 @@ public class INodeFile extends INodeWithAdditionalFields @Override public final ContentSummaryComputationContext computeContentSummary( int snapshotId, final ContentSummaryComputationContext summary) { + summary.nodeIncluded(this); final ContentCounts counts = summary.getCounts(); counts.addContent(Content.FILE, 1); final long fileLen = computeFileSize(snapshotId); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java index 1b852374c5d..56aaf8ce058 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java @@ -315,6 +315,7 @@ public abstract class INodeReference extends INode { @Override public ContentSummaryComputationContext computeContentSummary(int snapshotId, ContentSummaryComputationContext summary) { + summary.nodeIncluded(this); return referred.computeContentSummary(snapshotId, summary); } @@ -504,6 +505,7 @@ public abstract class INodeReference extends INode { @Override public final ContentSummaryComputationContext computeContentSummary( int snapshotId, ContentSummaryComputationContext summary) { + summary.nodeIncluded(this); final int s = snapshotId < lastSnapshotId ? snapshotId : lastSnapshotId; // only count storagespace for WithName final QuotaCounts q = computeQuotaUsage( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java index c76bea090f1..1223f4ef86c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java @@ -96,6 +96,7 @@ public class INodeSymlink extends INodeWithAdditionalFields { @Override public ContentSummaryComputationContext computeContentSummary(int snapshotId, final ContentSummaryComputationContext summary) { + summary.nodeIncluded(this); summary.getCounts().addContent(Content.SYMLINK, 1); return summary; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java index 8f65ff81ef3..f05fa377ef8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java @@ -278,6 +278,8 @@ public class INodesInPath { } private final byte[][] path; + private final String pathname; + /** * Array with the specified number of INodes resolved for a given path. */ @@ -306,6 +308,7 @@ public class INodesInPath { Preconditions.checkArgument(inodes != null && path != null); this.inodes = inodes; this.path = path; + this.pathname = DFSUtil.byteArray2PathString(path); this.isRaw = isRaw; this.isSnapshot = isSnapshot; this.snapshotId = snapshotId; @@ -366,7 +369,7 @@ public class INodesInPath { /** @return the full path in string form */ public String getPath() { - return DFSUtil.byteArray2PathString(path); + return pathname; } public String getParentPath() { @@ -399,7 +402,7 @@ public class INodesInPath { */ private INodesInPath getAncestorINodesInPath(int length) { Preconditions.checkArgument(length >= 0 && length < inodes.length); - Preconditions.checkState(!isSnapshot()); + Preconditions.checkState(isDotSnapshotDir() || !isSnapshot()); final INode[] anodes = new INode[length]; final byte[][] apath = new byte[length][]; System.arraycopy(this.inodes, 0, anodes, 0, length); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index ae7a9371240..afedbb9839a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -64,7 +64,9 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; +import org.apache.hadoop.ipc.ExternalCall; import org.apache.hadoop.ipc.RefreshCallQueueProtocol; +import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; @@ -407,7 +409,15 @@ public class NameNode extends ReconfigurableBase implements public NamenodeProtocols getRpcServer() { return rpcServer; } - + + public void queueExternalCall(ExternalCall extCall) + throws IOException, InterruptedException { + if (rpcServer == null) { + throw new RetriableException("Namenode is in startup mode"); + } + rpcServer.getClientRpcServer().queueCall(extCall); + } + public static void initMetrics(Configuration conf, NamenodeRole role) { metrics = NameNodeMetrics.create(conf, role); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 57f7cb197b6..a97a307aefd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -139,7 +139,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.IncorrectVersionException; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; -import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; @@ -1686,10 +1685,7 @@ public class NameNodeRpcServer implements NamenodeProtocols { } private static String getClientMachine() { - String clientMachine = NamenodeWebHdfsMethods.getRemoteAddress(); - if (clientMachine == null) { //not a web client - clientMachine = Server.getRemoteAddress(); - } + String clientMachine = Server.getRemoteAddress(); if (clientMachine == null) { //not a RPC client clientMachine = ""; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index d7c9a78ac23..83020353485 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -727,15 +727,30 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { String blkName = block.toString(); report.append(blockNumber + ". " + blkName + " len=" + block.getNumBytes()); - if (totalReplicasPerBlock == 0 && !isCorrupt) { + boolean isMissing; + if (storedBlock.isStriped()) { + isMissing = totalReplicasPerBlock < minReplication; + } else { + isMissing = totalReplicasPerBlock == 0; + } + if (isMissing && !isCorrupt) { // If the block is corrupted, it means all its available replicas are - // corrupted. We don't mark it as missing given these available replicas - // might still be accessible as the block might be incorrectly marked as - // corrupted by client machines. + // corrupted in the case of replication, and it means the state of the + // block group is unrecoverable due to some corrupted intenal blocks in + // the case of EC. We don't mark it as missing given these available + // replicas/internal-blocks might still be accessible as the block might + // be incorrectly marked as corrupted by client machines. report.append(" MISSING!"); res.addMissing(blkName, block.getNumBytes()); missing++; missize += block.getNumBytes(); + if (storedBlock.isStriped()) { + report.append(" Live_repl=" + liveReplicas); + String info = getReplicaInfo(storedBlock); + if (!info.isEmpty()){ + report.append(" ").append(info); + } + } } else { report.append(" Live_repl=" + liveReplicas); String info = getReplicaInfo(storedBlock); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java index 65a4ada3a7f..8a91e2a723b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java @@ -51,9 +51,10 @@ public class XAttrStorage { * @param prefixedName xAttr name with prefix * @return the xAttr */ - public static XAttr readINodeXAttrByPrefixedName(INode inode, - int snapshotId, String prefixedName) { - XAttrFeature f = inode.getXAttrFeature(snapshotId); + public static XAttr readINodeXAttrByPrefixedName(INodesInPath iip, + String prefixedName) { + XAttrFeature f = + iip.getLastINode().getXAttrFeature(iip.getPathSnapshotId()); return f == null ? null : f.getXAttr(prefixedName); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java index d8a516e13ea..945e92f68c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java @@ -31,14 +31,14 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.StandbyException; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.io.retry.MultiException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A FailoverProxyProvider implementation that technically does not "failover" @@ -51,8 +51,8 @@ import org.apache.hadoop.io.retry.MultiException; public class RequestHedgingProxyProvider extends ConfiguredFailoverProxyProvider { - private static final Log LOG = - LogFactory.getLog(RequestHedgingProxyProvider.class); + public static final Logger LOG = + LoggerFactory.getLogger(RequestHedgingProxyProvider.class); class RequestHedgingInvocationHandler implements InvocationHandler { @@ -100,6 +100,8 @@ public class RequestHedgingProxyProvider extends Callable c = new Callable() { @Override public Object call() throws Exception { + LOG.trace("Invoking method {} on proxy {}", method, + pEntry.getValue().proxyInfo); return method.invoke(pEntry.getValue().proxy, args); } }; @@ -114,15 +116,14 @@ public class RequestHedgingProxyProvider extends try { retVal = callResultFuture.get(); successfulProxy = proxyMap.get(callResultFuture); - if (LOG.isDebugEnabled()) { - LOG.debug("Invocation successful on [" - + successfulProxy.proxyInfo + "]"); - } + LOG.debug("Invocation successful on [{}]", + successfulProxy.proxyInfo); return retVal; } catch (Exception ex) { ProxyInfo tProxyInfo = proxyMap.get(callResultFuture); logProxyException(ex, tProxyInfo.proxyInfo); badResults.put(tProxyInfo.proxyInfo, ex); + LOG.trace("Unsuccessful invocation on [{}]", tProxyInfo.proxyInfo); numAttempts--; } } @@ -136,6 +137,7 @@ public class RequestHedgingProxyProvider extends } } finally { if (executor != null) { + LOG.trace("Shutting down threadpool executor"); executor.shutdownNow(); } } @@ -193,12 +195,9 @@ public class RequestHedgingProxyProvider extends */ private void logProxyException(Exception ex, String proxyInfo) { if (isStandbyException(ex)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Invocation returned standby exception on [" + - proxyInfo + "]"); - } + LOG.debug("Invocation returned standby exception on [{}]", proxyInfo); } else { - LOG.warn("Invocation returned exception on [" + proxyInfo + "]"); + LOG.warn("Invocation returned exception on [{}]", proxyInfo); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java index 39db979ae02..fa7bacee84d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java @@ -29,9 +29,9 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.SnapshotException; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.namenode.Content; import org.apache.hadoop.hdfs.server.namenode.ContentCounts; +import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.SnapshotAndINode; @@ -220,11 +220,12 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature } @Override - public void computeContentSummary4Snapshot(final BlockStoragePolicySuite bsps, - final ContentCounts counts) { + public void computeContentSummary4Snapshot(ContentSummaryComputationContext + context) { + ContentCounts counts = context.getCounts(); counts.addContent(Content.SNAPSHOT, snapshotsByNames.size()); counts.addContent(Content.SNAPSHOTTABLE_DIRECTORY, 1); - super.computeContentSummary4Snapshot(bsps, counts); + super.computeContentSummary4Snapshot(context); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index 0111b3bfb65..9addbfa7ace 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -30,7 +30,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.namenode.AclStorage; -import org.apache.hadoop.hdfs.server.namenode.ContentCounts; import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext; import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; import org.apache.hadoop.hdfs.server.namenode.INode; @@ -629,18 +628,13 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { return counts; } - public void computeContentSummary4Snapshot(final BlockStoragePolicySuite bsps, - final ContentCounts counts) { - // Create a new blank summary context for blocking processing of subtree. - ContentSummaryComputationContext summary = - new ContentSummaryComputationContext(bsps); + public void computeContentSummary4Snapshot( + ContentSummaryComputationContext context) { for(DirectoryDiff d : diffs) { - for(INode deleted : d.getChildrenDiff().getList(ListType.DELETED)) { - deleted.computeContentSummary(Snapshot.CURRENT_STATE_ID, summary); + for(INode deletedNode : d.getChildrenDiff().getList(ListType.DELETED)) { + context.reportDeletedSnapshottedNode(deletedNode); } } - // Add the counts from deleted trees. - counts.addContents(summary.getCounts()); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java index e98e766d34a..832a3391f5b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java @@ -177,6 +177,7 @@ public class Snapshot implements Comparable { @Override public ContentSummaryComputationContext computeContentSummary( int snapshotId, ContentSummaryComputationContext summary) { + summary.nodeIncluded(this); return computeDirectoryContentSummary(summary, snapshotId); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java index ab553928862..2719c8857ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java @@ -17,24 +17,32 @@ */ package org.apache.hadoop.hdfs.server.namenode.top.metrics; -import java.net.InetAddress; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - import com.google.common.collect.Lists; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.server.namenode.top.TopConf; import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager; +import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.Op; +import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.User; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.lib.Interns; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.InetAddress; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.TopWindow; /** @@ -58,8 +66,11 @@ import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowMan * Thread-safe: relies on thread-safety of RollingWindowManager */ @InterfaceAudience.Private -public class TopMetrics { +public class TopMetrics implements MetricsSource { public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class); + public static final String TOPMETRICS_METRICS_SOURCE_NAME = + "NNTopUserOpCounts"; + private final boolean isMetricsSourceEnabled; private static void logConf(Configuration conf) { LOG.info("NNTop conf: " + DFSConfigKeys.NNTOP_BUCKETS_PER_WINDOW_KEY + @@ -83,6 +94,8 @@ public class TopMetrics { rollingWindowManagers.put(reportingPeriods[i], new RollingWindowManager( conf, reportingPeriods[i])); } + isMetricsSourceEnabled = conf.getBoolean(DFSConfigKeys.NNTOP_ENABLED_KEY, + DFSConfigKeys.NNTOP_ENABLED_DEFAULT); } /** @@ -128,4 +141,44 @@ public class TopMetrics { TopConf.ALL_CMDS, userName, 1); } } + + /** + * Flatten out the top window metrics into + * {@link org.apache.hadoop.metrics2.MetricsRecord}s for consumption by + * external metrics systems. Each metrics record added corresponds to the + * reporting period a.k.a window length of the configured rolling windows. + */ + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + if (!isMetricsSourceEnabled) { + return; + } + + for (final TopWindow window : getTopWindows()) { + MetricsRecordBuilder rb = collector.addRecord(buildOpRecordName(window)) + .setContext("dfs"); + for (final Op op: window.getOps()) { + rb.addCounter(buildOpTotalCountMetricsInfo(op), op.getTotalCount()); + for (User user : op.getTopUsers()) { + rb.addCounter(buildOpRecordMetricsInfo(op, user), user.getCount()); + } + } + } + } + + private String buildOpRecordName(TopWindow window) { + return TOPMETRICS_METRICS_SOURCE_NAME + ".windowMs=" + + window.getWindowLenMs(); + } + + private MetricsInfo buildOpTotalCountMetricsInfo(Op op) { + return Interns.info("op=" + StringUtils.deleteWhitespace(op.getOpType()) + + ".TotalCount", "Total operation count"); + } + + private MetricsInfo buildOpRecordMetricsInfo(Op op, User user) { + return Interns.info("op=" + StringUtils.deleteWhitespace(op.getOpType()) + + ".user=" + user.getUser() + + ".count", "Total operations performed by user"); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 3ab0c676e85..4247a673bcf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -25,10 +25,13 @@ import java.io.PrintWriter; import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.security.Principal; import java.security.PrivilegedExceptionAction; import java.util.EnumSet; import java.util.HashSet; import java.util.List; +import java.util.concurrent.ExecutionException; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; @@ -60,6 +63,7 @@ import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsCreateModes; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -81,8 +85,8 @@ import org.apache.hadoop.hdfs.web.WebHdfsConstants; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.*; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ipc.ExternalCall; import org.apache.hadoop.ipc.RetriableException; -import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.security.Credentials; @@ -103,39 +107,39 @@ public class NamenodeWebHdfsMethods { public static final Log LOG = LogFactory.getLog(NamenodeWebHdfsMethods.class); private static final UriFsPathParam ROOT = new UriFsPathParam(""); - - private static final ThreadLocal REMOTE_ADDRESS = new ThreadLocal(); - /** @return the remote client address. */ - public static String getRemoteAddress() { - return REMOTE_ADDRESS.get(); - } - - public static InetAddress getRemoteIp() { - try { - return InetAddress.getByName(getRemoteAddress()); - } catch (Exception e) { - return null; - } - } - - /** - * Returns true if a WebHdfs request is in progress. Akin to - * {@link Server#isRpcInvocation()}. - */ - public static boolean isWebHdfsInvocation() { - return getRemoteAddress() != null; - } + private volatile Boolean useIpcCallq; + private String scheme; + private Principal userPrincipal; + private String remoteAddr; private @Context ServletContext context; - private @Context HttpServletRequest request; private @Context HttpServletResponse response; + public NamenodeWebHdfsMethods(@Context HttpServletRequest request) { + // the request object is a proxy to thread-locals so we have to extract + // what we want from it since the external call will be processed in a + // different thread. + scheme = request.getScheme(); + userPrincipal = request.getUserPrincipal(); + // get the remote address, if coming in via a trusted proxy server then + // the address with be that of the proxied client + remoteAddr = JspHelper.getRemoteAddr(request); + } + private void init(final UserGroupInformation ugi, final DelegationParam delegation, final UserParam username, final DoAsParam doAsUser, final UriFsPathParam path, final HttpOpParam op, final Param... parameters) { + if (useIpcCallq == null) { + Configuration conf = + (Configuration)context.getAttribute(JspHelper.CURRENT_CONF); + useIpcCallq = conf.getBoolean( + DFSConfigKeys.DFS_WEBHDFS_USE_IPC_CALLQ, + DFSConfigKeys.DFS_WEBHDFS_USE_IPC_CALLQ_DEFAULT); + } + if (LOG.isTraceEnabled()) { LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path + ", ugi=" + ugi + ", " + username + ", " + doAsUser @@ -144,16 +148,8 @@ public class NamenodeWebHdfsMethods { //clear content type response.setContentType(null); - - // set the remote address, if coming in via a trust proxy server then - // the address with be that of the proxied client - REMOTE_ADDRESS.set(JspHelper.getRemoteAddr(request)); } - private void reset() { - REMOTE_ADDRESS.set(null); - } - private static NamenodeProtocols getRPCServer(NameNode namenode) throws IOException { final NamenodeProtocols np = namenode.getRpcServer(); @@ -162,11 +158,63 @@ public class NamenodeWebHdfsMethods { } return np; } - + + private T doAs(final UserGroupInformation ugi, + final PrivilegedExceptionAction action) + throws IOException, InterruptedException { + return useIpcCallq ? doAsExternalCall(ugi, action) : ugi.doAs(action); + } + + private T doAsExternalCall(final UserGroupInformation ugi, + final PrivilegedExceptionAction action) + throws IOException, InterruptedException { + // set the remote address, if coming in via a trust proxy server then + // the address with be that of the proxied client + ExternalCall call = new ExternalCall(action){ + @Override + public UserGroupInformation getRemoteUser() { + return ugi; + } + @Override + public String getProtocol() { + return "webhdfs"; + } + @Override + public String getHostAddress() { + return remoteAddr; + } + @Override + public InetAddress getHostInetAddress() { + try { + return InetAddress.getByName(getHostAddress()); + } catch (UnknownHostException e) { + return null; + } + } + }; + final NameNode namenode = (NameNode)context.getAttribute("name.node"); + namenode.queueExternalCall(call); + T result = null; + try { + result = call.get(); + } catch (ExecutionException ee) { + Throwable t = ee.getCause(); + if (t instanceof RuntimeException) { + throw (RuntimeException)t; + } else if (t instanceof IOException) { + throw (IOException)t; + } else { + throw new IOException(t); + } + } + return result; + } + @VisibleForTesting static DatanodeInfo chooseDatanode(final NameNode namenode, final String path, final HttpOpParam.Op op, final long openOffset, - final long blocksize, final String excludeDatanodes) throws IOException { + final long blocksize, final String excludeDatanodes, + final String remoteAddr) throws IOException { FSNamesystem fsn = namenode.getNamesystem(); if (fsn == null) { throw new IOException("Namesystem has not been intialized yet."); @@ -190,7 +238,7 @@ public class NamenodeWebHdfsMethods { if (op == PutOpParam.Op.CREATE) { //choose a datanode near to client final DatanodeDescriptor clientNode = bm.getDatanodeManager( - ).getDatanodeByHost(getRemoteAddress()); + ).getDatanodeByHost(remoteAddr); if (clientNode != null) { final DatanodeStorageInfo[] storages = bm.chooseTarget4WebHDFS( path, clientNode, excludes, blocksize); @@ -253,7 +301,8 @@ public class NamenodeWebHdfsMethods { return null; } final Token t = c.getAllTokens().iterator().next(); - Text kind = request.getScheme().equals("http") ? WebHdfsConstants.WEBHDFS_TOKEN_KIND + Text kind = scheme.equals("http") + ? WebHdfsConstants.WEBHDFS_TOKEN_KIND : WebHdfsConstants.SWEBHDFS_TOKEN_KIND; t.setKind(kind); return t; @@ -267,7 +316,7 @@ public class NamenodeWebHdfsMethods { final Param... parameters) throws URISyntaxException, IOException { final DatanodeInfo dn; dn = chooseDatanode(namenode, path, op, openOffset, blocksize, - excludeDatanodes); + excludeDatanodes, remoteAddr); if (dn == null) { throw new IOException("Failed to find datanode, suggest to check cluster" + " health. excludeDatanodes=" + excludeDatanodes); @@ -283,7 +332,7 @@ public class NamenodeWebHdfsMethods { } else { //generate a token final Token t = generateDelegationToken( - namenode, ugi, request.getUserPrincipal().getName()); + namenode, ugi, null); delegationQuery = "&" + new DelegationParam(t.encodeToUrlString()); } final String query = op.toQueryString() + delegationQuery @@ -291,7 +340,6 @@ public class NamenodeWebHdfsMethods { + Param.toSortedString("&", parameters); final String uripath = WebHdfsFileSystem.PATH_PREFIX + path; - final String scheme = request.getScheme(); int port = "http".equals(scheme) ? dn.getInfoPort() : dn .getInfoSecurePort(); final URI uri = new URI(scheme, null, dn.getHostName(), port, uripath, @@ -446,10 +494,9 @@ public class NamenodeWebHdfsMethods { xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes, createFlagParam, noredirect); - return ugi.doAs(new PrivilegedExceptionAction() { + return doAs(ugi, new PrivilegedExceptionAction() { @Override public Response run() throws IOException, URISyntaxException { - try { return put(ugi, delegation, username, doAsUser, path.getAbsolutePath(), op, destination, owner, group, permission, unmaskedPermission, overwrite, bufferSize, @@ -458,9 +505,6 @@ public class NamenodeWebHdfsMethods { aclPermission, xattrName, xattrValue, xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes, createFlagParam, noredirect); - } finally { - reset(); - } } }); } @@ -703,16 +747,12 @@ public class NamenodeWebHdfsMethods { init(ugi, delegation, username, doAsUser, path, op, concatSrcs, bufferSize, excludeDatanodes, newLength); - return ugi.doAs(new PrivilegedExceptionAction() { + return doAs(ugi, new PrivilegedExceptionAction() { @Override public Response run() throws IOException, URISyntaxException { - try { return post(ugi, delegation, username, doAsUser, path.getAbsolutePath(), op, concatSrcs, bufferSize, excludeDatanodes, newLength, noredirect); - } finally { - reset(); - } } }); } @@ -858,17 +898,13 @@ public class NamenodeWebHdfsMethods { renewer, bufferSize, xattrEncoding, excludeDatanodes, fsAction, tokenKind, tokenService, startAfter); - return ugi.doAs(new PrivilegedExceptionAction() { + return doAs(ugi, new PrivilegedExceptionAction() { @Override public Response run() throws IOException, URISyntaxException { - try { return get(ugi, delegation, username, doAsUser, path.getAbsolutePath(), op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, fsAction, tokenKind, tokenService, noredirect, startAfter); - } finally { - reset(); - } } }); } @@ -1138,15 +1174,11 @@ public class NamenodeWebHdfsMethods { init(ugi, delegation, username, doAsUser, path, op, recursive, snapshotName); - return ugi.doAs(new PrivilegedExceptionAction() { + return doAs(ugi, new PrivilegedExceptionAction() { @Override public Response run() throws IOException { - try { return delete(ugi, delegation, username, doAsUser, path.getAbsolutePath(), op, recursive, snapshotName); - } finally { - reset(); - } } }); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java index 06389a159f0..b78da31e78f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; @@ -193,6 +194,53 @@ public class CryptoAdmin extends Configured implements Tool { } } + private static class GetFileEncryptionInfoCommand + implements AdminHelper.Command { + @Override + public String getName() { + return "-getFileEncryptionInfo"; + } + + @Override + public String getShortUsage() { + return "[" + getName() + " -path ]\n"; + } + + @Override + public String getLongUsage() { + final TableListing listing = AdminHelper.getOptionDescriptionListing(); + listing.addRow("", "The path to the file to show encryption info."); + return getShortUsage() + "\n" + "Get encryption info of a file.\n\n" + + listing.toString(); + } + + @Override + public int run(Configuration conf, List args) throws IOException { + final String path = StringUtils.popOptionWithArgument("-path", args); + + if (!args.isEmpty()) { + System.err.println("Can't understand argument: " + args.get(0)); + return 1; + } + + final HdfsAdmin admin = + new HdfsAdmin(FileSystem.getDefaultUri(conf), conf); + try { + final FileEncryptionInfo fei = + admin.getFileEncryptionInfo(new Path(path)); + if (fei == null) { + System.out.println("No FileEncryptionInfo found for path " + path); + return 2; + } + System.out.println(fei.toStringStable()); + } catch (IOException e) { + System.err.println(prettifyException(e)); + return 3; + } + return 0; + } + } + private static class ProvisionTrashCommand implements AdminHelper.Command { @Override public String getName() { @@ -237,6 +285,7 @@ public class CryptoAdmin extends Configured implements Tool { private static final AdminHelper.Command[] COMMANDS = { new CreateZoneCommand(), new ListZonesCommand(), - new ProvisionTrashCommand() + new ProvisionTrashCommand(), + new GetFileEncryptionInfoCommand() }; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index e6e0fbb3064..a60f24b1aef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -716,7 +716,7 @@ public class DFSAdmin extends FsShell { } /** - * Allow snapshot on a directory. + * Disallow snapshot on a directory. * Usage: hdfs dfsadmin -disallowSnapshot snapshotDir * @param argv List of of command line parameters. * @exception IOException @@ -936,8 +936,7 @@ public class DFSAdmin extends FsShell { System.out.println("Balancer bandwidth is " + bandwidth + " bytes per second."); } catch (IOException ioe) { - System.err.println("Datanode unreachable."); - return -1; + throw new IOException("Datanode unreachable. " + ioe, ioe); } return 0; } @@ -2207,7 +2206,7 @@ public class DFSAdmin extends FsShell { dnProxy.evictWriters(); System.out.println("Requested writer eviction to datanode " + dn); } catch (IOException ioe) { - return -1; + throw new IOException("Datanode unreachable. " + ioe, ioe); } return 0; } @@ -2218,8 +2217,7 @@ public class DFSAdmin extends FsShell { DatanodeLocalInfo dnInfo = dnProxy.getDatanodeInfo(); System.out.println(dnInfo.getDatanodeLocalReport()); } catch (IOException ioe) { - System.err.println("Datanode unreachable."); - return -1; + throw new IOException("Datanode unreachable. " + ioe, ioe); } return 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 6b6cca62f01..0542f3a5c35 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -360,7 +360,7 @@ public class JsonUtil { final List stringEntries = new ArrayList<>(); for (AclEntry entry : status.getEntries()) { - stringEntries.add(entry.toString()); + stringEntries.add(entry.toStringStable()); } m.put("entries", stringEntries); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index ebaefde2622..84b51f6159b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -725,7 +725,7 @@ Setting this limit to 1000 disables compiler thread throttling. Only values between 1 and 1000 are valid. Setting an invalid value will result - in the throttle being disbled and an error message being logged. 1000 is + in the throttle being disabled and an error message being logged. 1000 is the default setting. @@ -2559,7 +2559,7 @@ dfs.block.local-path-access.user - Comma separated list of the users allowd to open block files + Comma separated list of the users allowed to open block files on legacy short-circuit local read. @@ -3650,7 +3650,7 @@ dfs.datanode.transferTo.allowed true - If false, break block tranfers on 32-bit machines greater than + If false, break block transfers on 32-bit machines greater than or equal to 2GB into smaller chunks. @@ -4273,4 +4273,19 @@ consecutive warnings within this interval. + + httpfs.buffer.size + 4096 + + The size buffer to be used when creating or opening httpfs filesystem IO stream. + + + + + dfs.webhdfs.use.ipc.callq + true + Enables routing of webhdfs calls through rpc + call queue + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ExtendedAttributes.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ExtendedAttributes.md index 5a2098633e4..eb527abafd7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ExtendedAttributes.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ExtendedAttributes.md @@ -30,7 +30,7 @@ Overview ### HDFS extended attributes -Extended attributes in HDFS are modeled after extended attributes in Linux (see the Linux manpage for [attr(5)](http://www.bestbits.at/acl/man/man5/attr.txt) and [related documentation](http://www.bestbits.at/acl/)). An extended attribute is a *name-value pair*, with a string name and binary value. Xattrs names must also be prefixed with a *namespace*. For example, an xattr named *myXattr* in the *user* namespace would be specified as **user.myXattr**. Multiple xattrs can be associated with a single inode. +Extended attributes in HDFS are modeled after extended attributes in Linux (see the Linux manpage for [attr(5)](http://man7.org/linux/man-pages/man5/attr.5.html)). An extended attribute is a *name-value pair*, with a string name and binary value. Xattrs names must also be prefixed with a *namespace*. For example, an xattr named *myXattr* in the *user* namespace would be specified as **user.myXattr**. Multiple xattrs can be associated with a single inode. ### Namespaces and Permissions @@ -49,7 +49,7 @@ The `raw` namespace is reserved for internal system attributes that sometimes ne Interacting with extended attributes ------------------------------------ -The Hadoop shell has support for interacting with extended attributes via `hadoop fs -getfattr` and `hadoop fs -setfattr`. These commands are styled after the Linux [getfattr(1)](http://www.bestbits.at/acl/man/man1/getfattr.txt) and [setfattr(1)](http://www.bestbits.at/acl/man/man1/setfattr.txt) commands. +The Hadoop shell has support for interacting with extended attributes via `hadoop fs -getfattr` and `hadoop fs -setfattr`. These commands are styled after the Linux [getfattr(1)](http://man7.org/linux/man-pages/man1/getfattr.1.html) and [setfattr(1)](http://man7.org/linux/man-pages/man1/setfattr.1.html) commands. ### getfattr diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index 9f9fba5d2fb..e923b86d0f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -463,7 +463,7 @@ Runs the diskbalancer CLI. See [HDFS Diskbalancer](./HDFSDiskbalancer.html) for Usage: hdfs erasurecode [generic options] - [-setPolicy [-s ] ] + [-setPolicy [-p ] ] [-getPolicy ] [-listPolicies] [-usage [cmd ...]] diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 9066a1561ae..627260f8792 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -22,6 +22,7 @@ HDFS Erasure Coding * [Deployment](#Deployment) * [Cluster and hardware configuration](#Cluster_and_hardware_configuration) * [Configuration keys](#Configuration_keys) + * [Enable Intel ISA-L](#Enable_Intel_ISA-L) * [Administrative commands](#Administrative_commands) Purpose @@ -59,9 +60,9 @@ Architecture 1. _Read the data from source nodes:_ Input data is read in parallel from source nodes using a dedicated thread pool. Based on the EC policy, it schedules the read requests to all source targets and reads only the minimum number of input blocks for reconstruction. - 1. _Decode the data and generate the output data:_ New data and parity blocks are decoded from the input data. All missing data and parity blocks are decoded together. + 2. _Decode the data and generate the output data:_ New data and parity blocks are decoded from the input data. All missing data and parity blocks are decoded together. - 1. _Transfer the generated data blocks to target nodes:_ Once decoding is finished, the recovered blocks are transferred to target DataNodes. + 3. _Transfer the generated data blocks to target nodes:_ Once decoding is finished, the recovered blocks are transferred to target DataNodes. * **ErasureCoding policy** To accommodate heterogeneous workloads, we allow files and directories in an HDFS cluster to have different replication and EC policies. @@ -69,11 +70,13 @@ Architecture 1. _The ECSchema:_ This includes the numbers of data and parity blocks in an EC group (e.g., 6+3), as well as the codec algorithm (e.g., Reed-Solomon). - 1. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work. + 2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work. - Currently, HDFS supports the Reed-Solomon and XOR erasure coding algorithms. Additional algorithms are planned as future work. - The system default scheme is Reed-Solomon (6, 3) with a cell size of 64KB. + There are three policies currently being supported: RS-DEFAULT-3-2-64k, RS-DEFAULT-6-3-64k and RS-LEGACY-6-3-64k. All with default cell size of 64KB. The system default policy is RS-DEFAULT-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB. + * **Intel ISA-L** + Intel ISA-L stands for Intel Intelligent Storage Acceleration Library. ISA-L is a collection of optimized low-level functions used primarily in storage applications. It includes a fast block Reed-Solomon type erasure codes optimized for Intel AVX and AVX2 instruction sets. + HDFS EC can leverage this open-source library to accelerate encoding and decoding calculation. ISA-L supports most of major operating systems, including Linux and Windows. By default, ISA-L is not enabled in HDFS. Deployment ---------- @@ -99,7 +102,7 @@ Deployment `io.erasurecode.codec.rs-default.rawcoder` for the default RS codec, `io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec, `io.erasurecode.codec.xor.rawcoder` for the XOR codec. - The default implementations for all of these codecs are pure Java. + The default implementations for all of these codecs are pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the encoding and decoding calculation. Please refer to section "Enable Intel ISA-L" for more detail information. Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters: @@ -107,6 +110,15 @@ Deployment 1. `dfs.datanode.stripedread.threads` - Number of concurrent reader threads. Default value is 20 threads. 1. `dfs.datanode.stripedread.buffer.size` - Buffer size for reader service. Default value is 256KB. +### Enable Intel ISA-L + + HDFS native implementation of default RS codec leverages Intel ISA-L library to improve the encoding and decoding calculation. To enable and use Intel ISA-L, there are three steps. + 1. Build ISA-L library. Please refer to the offical site "https://github.com/01org/isa-l/" for detail information. + 2. Build Hadoop with ISA-L support. Please refer to "Intel ISA-L build options" section in "Build instructions for Hadoop"(BUILDING.txt) document. Use -Dbundle.isal to copy the contents of the isal.lib directory into the final tar file. Deploy hadoop with the tar file. Make sure ISA-L library is available on both HDFS client and DataNodes. + 3. Configure the `io.erasurecode.codec.rs-default.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes. + + To check ISA-L library enable state, try "Hadoop checknative" command. It will tell you if ISA-L library is enabled or not. + ### Administrative commands HDFS provides an `erasurecode` subcommand to perform administrative commands related to erasure coding. @@ -126,7 +138,7 @@ Below are the details about each command. `path`: An directory in HDFS. This is a mandatory parameter. Setting a policy only affects newly created files, and does not affect existing files. - `policyName`: The ErasureCoding policy to be used for files under this directory. This is an optional parameter, specified using ‘-s’ flag. If no policy is specified, the system default ErasureCodingPolicy will be used. + `policyName`: The ErasureCoding policy to be used for files under this directory. This is an optional parameter, specified using ‘-p’ flag. If no policy is specified, the system default ErasureCodingPolicy will be used. * `[-getPolicy ]` diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md index b743233c100..5625244835b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md @@ -38,7 +38,6 @@ HDFS High Availability * [Securing access to ZooKeeper](#Securing_access_to_ZooKeeper) * [Verifying automatic failover](#Verifying_automatic_failover) * [Automatic Failover FAQ](#Automatic_Failover_FAQ) - * [BookKeeper as a Shared storage (EXPERIMENTAL)](#BookKeeper_as_a_Shared_storage_EXPERIMENTAL) Purpose ------- @@ -572,116 +571,3 @@ Automatic Failover FAQ using the same `hdfs haadmin` command. It will perform a coordinated failover. -BookKeeper as a Shared storage (EXPERIMENTAL) ---------------------------------------------- - -One option for shared storage for the NameNode is BookKeeper. BookKeeper achieves high availability and strong durability guarantees by replicating edit log entries across multiple storage nodes. The edit log can be striped across the storage nodes for high performance. Fencing is supported in the protocol, i.e, BookKeeper will not allow two writers to write the single edit log. - -The meta data for BookKeeper is stored in ZooKeeper. In current HA architecture, a Zookeeper cluster is required for ZKFC. The same cluster can be for BookKeeper metadata. - -For more details on building a BookKeeper cluster, please refer to the [BookKeeper documentation](http://zookeeper.apache.org/bookkeeper/docs/trunk/bookkeeperConfig.html ) - -The BookKeeperJournalManager is an implementation of the HDFS JournalManager interface, which allows custom write ahead logging implementations to be plugged into the HDFS NameNode. - -* **BookKeeper Journal Manager** - - To use BookKeeperJournalManager, add the following to hdfs-site.xml. - - - dfs.namenode.shared.edits.dir - bookkeeper://zk1:2181;zk2:2181;zk3:2181/hdfsjournal - - - - dfs.namenode.edits.journal-plugin.bookkeeper - org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager - - - The URI format for bookkeeper is `bookkeeper://[zkEnsemble]/[rootZnode] [zookkeeper ensemble]` - is a list of semi-colon separated, zookeeper host:port - pairs. In the example above there are 3 servers, in the ensemble, - zk1, zk2 & zk3, each one listening on port 2181. - - `[root znode]` is the path of the zookeeper znode, under which the edit log - information will be stored. - - The class specified for the journal-plugin must be available in the NameNode's - classpath. We explain how to generate a jar file with the journal manager and - its dependencies, and how to put it into the classpath below. - -* **More configuration options** - - * **dfs.namenode.bookkeeperjournal.output-buffer-size** - - Number of bytes a bookkeeper journal stream will buffer before - forcing a flush. Default is 1024. - - - dfs.namenode.bookkeeperjournal.output-buffer-size - 1024 - - - * **dfs.namenode.bookkeeperjournal.ensemble-size** - - Number of bookkeeper servers in edit log ensembles. This - is the number of bookkeeper servers which need to be available - for the edit log to be writable. Default is 3. - - - dfs.namenode.bookkeeperjournal.ensemble-size - 3 - - - * **dfs.namenode.bookkeeperjournal.quorum-size** - - Number of bookkeeper servers in the write quorum. This is the - number of bookkeeper servers which must have acknowledged the - write of an entry before it is considered written. Default is 2. - - - dfs.namenode.bookkeeperjournal.quorum-size - 2 - - - * **dfs.namenode.bookkeeperjournal.digestPw** - - Password to use when creating edit log segments. - - - dfs.namenode.bookkeeperjournal.digestPw - myPassword - - - * **dfs.namenode.bookkeeperjournal.zk.session.timeout** - - Session timeout for Zookeeper client from BookKeeper Journal Manager. - Hadoop recommends that this value should be less than the ZKFC - session timeout value. Default value is 3000. - - - dfs.namenode.bookkeeperjournal.zk.session.timeout - 3000 - - -* **Building BookKeeper Journal Manager plugin jar** - - To generate the distribution packages for BK journal, do the following. - - $ mvn clean package -Pdist - - This will generate a jar with the BookKeeperJournalManager, - hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-*VERSION*.jar - - Note that the -Pdist part of the build command is important, this would - copy the dependent bookkeeper-server jar under - hadoop-hdfs/src/contrib/bkjournal/target/lib. - -* **Putting the BookKeeperJournalManager in the NameNode classpath** - - To run a HDFS namenode using BookKeeper as a backend, copy the bkjournal and - bookkeeper-server jar, mentioned above, into the lib directory of hdfs. In the - standard distribution of HDFS, this is at $HADOOP\_HDFS\_HOME/share/hadoop/hdfs/lib/ - - cp hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-*VERSION*.jar $HADOOP\_HDFS\_HOME/share/hadoop/hdfs/lib/ - -* **Current limitations** - - 1) Security in BookKeeper. BookKeeper does not support SASL nor SSL for - connections between the NameNode and BookKeeper storage nodes. - - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md index ee98df8308f..e7d9f1d20d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md @@ -29,6 +29,8 @@ Transparent Encryption in HDFS * [crypto command-line interface](#crypto_command-line_interface) * [createZone](#createZone) * [listZones](#listZones) + * [provisionTrash](#provisionTrash) + * [getFileEncryptionInfo](#getFileEncryptionInfo) * [Example usage](#Example_usage) * [Distcp considerations](#Distcp_considerations) * [Running as the superuser](#Running_as_the_superuser) @@ -189,6 +191,16 @@ Provision a trash directory for an encryption zone. |:---- |:---- | | *path* | The path to the root of the encryption zone. | +### getFileEncryptionInfo + +Usage: `[-getFileEncryptionInfo -path ]` + +Get encryption information from a file. This can be used to find out whether a file is being encrypted, and the key name / key version used to encrypt it. + +| | | +|:---- |:---- | +| *path* | The path of the file to get encryption information. | + Example usage ------------- @@ -208,6 +220,10 @@ These instructions assume that you are running as the normal user or HDFS superu hadoop fs -put helloWorld /zone hadoop fs -cat /zone/helloWorld + # As the normal user, get encryption information from the file + hdfs crypto -getFileEncryptionInfo -path /zone/helloWorld + # console output: {cipherSuite: {name: AES/CTR/NoPadding, algorithmBlockSize: 16}, cryptoProtocolVersion: CryptoProtocolVersion{description='Encryption zones', version=1, unknownValue=null}, edek: 2010d301afbd43b58f10737ce4e93b39, iv: ade2293db2bab1a2e337f91361304cb3, keyName: mykey, ezKeyVersionName: mykey@0} + Distcp considerations --------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md index 546f99e96b5..f904bda3a5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md @@ -172,7 +172,7 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop * HTTP POST * [`APPEND`](#Append_to_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).append) * [`CONCAT`](#Concat_Files) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).concat) - * [`TRUNCATE`](#Truncate_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).concat) + * [`TRUNCATE`](#Truncate_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).truncate) * HTTP DELETE * [`DELETE`](#Delete_a_FileDirectory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).delete) * [`DELETESNAPSHOT`](#Delete_Snapshot) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).deleteSnapshot) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java index 0ccc07a833d..9cd46c191d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.TimeoutException; +import org.apache.commons.collections.map.LinkedMap; import org.apache.commons.lang.SystemUtils; import org.apache.commons.lang.mutable.MutableBoolean; import org.apache.commons.logging.Log; @@ -307,8 +308,8 @@ public class TestEnhancedByteBufferAccess { public void visit(int numOutstandingMmaps, Map replicas, Map failedLoads, - Map evictable, - Map evictableMmapped) { + LinkedMap evictable, + LinkedMap evictableMmapped) { if (expectedNumOutstandingMmaps >= 0) { Assert.assertEquals(expectedNumOutstandingMmaps, numOutstandingMmaps); } @@ -373,8 +374,8 @@ public class TestEnhancedByteBufferAccess { public void visit(int numOutstandingMmaps, Map replicas, Map failedLoads, - Map evictable, - Map evictableMmapped) { + LinkedMap evictable, + LinkedMap evictableMmapped) { ShortCircuitReplica replica = replicas.get( new ExtendedBlockId(firstBlock.getBlockId(), firstBlock.getBlockPoolId())); Assert.assertNotNull(replica); @@ -410,8 +411,8 @@ public class TestEnhancedByteBufferAccess { public void visit(int numOutstandingMmaps, Map replicas, Map failedLoads, - Map evictable, - Map evictableMmapped) { + LinkedMap evictable, + LinkedMap evictableMmapped) { finished.setValue(evictableMmapped.isEmpty()); } }); @@ -685,8 +686,8 @@ public class TestEnhancedByteBufferAccess { public void visit(int numOutstandingMmaps, Map replicas, Map failedLoads, - Map evictable, - Map evictableMmapped) { + LinkedMap evictable, + LinkedMap evictableMmapped) { Assert.assertEquals(expectedOutstandingMmaps, numOutstandingMmaps); ShortCircuitReplica replica = replicas.get(ExtendedBlockId.fromExtendedBlock(block)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBalancerBandwidth.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBalancerBandwidth.java index 6e6bbeef228..6bbe3a10bcc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBalancerBandwidth.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBalancerBandwidth.java @@ -24,13 +24,15 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.concurrent.TimeoutException; +import com.google.common.base.Supplier; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.tools.DFSAdmin; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; /** @@ -54,9 +56,8 @@ public class TestBalancerBandwidth { DEFAULT_BANDWIDTH); /* Create and start cluster */ - MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(NUM_OF_DATANODES).build(); - try { + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(NUM_OF_DATANODES).build()) { cluster.waitActive(); DistributedFileSystem fs = cluster.getFileSystem(); @@ -65,12 +66,6 @@ public class TestBalancerBandwidth { // Ensure value from the configuration is reflected in the datanodes. assertEquals(DEFAULT_BANDWIDTH, (long) datanodes.get(0).getBalancerBandwidth()); assertEquals(DEFAULT_BANDWIDTH, (long) datanodes.get(1).getBalancerBandwidth()); - ClientDatanodeProtocol dn1Proxy = DFSUtilClient - .createClientDatanodeProtocolProxy(datanodes.get(0).getDatanodeId(), - conf, 60000, false); - ClientDatanodeProtocol dn2Proxy = DFSUtilClient - .createClientDatanodeProtocolProxy(datanodes.get(1).getDatanodeId(), - conf, 60000, false); DFSAdmin admin = new DFSAdmin(conf); String dn1Address = datanodes.get(0).ipcServer.getListenerAddress() .getHostName() + ":" + datanodes.get(0).getIpcPort(); @@ -79,51 +74,49 @@ public class TestBalancerBandwidth { // verifies the dfsadmin command execution String[] args = new String[] { "-getBalancerBandwidth", dn1Address }; - runGetBalancerBandwidthCmd(admin, args, dn1Proxy, DEFAULT_BANDWIDTH); + runGetBalancerBandwidthCmd(admin, args, DEFAULT_BANDWIDTH); args = new String[] { "-getBalancerBandwidth", dn2Address }; - runGetBalancerBandwidthCmd(admin, args, dn2Proxy, DEFAULT_BANDWIDTH); + runGetBalancerBandwidthCmd(admin, args, DEFAULT_BANDWIDTH); // Dynamically change balancer bandwidth and ensure the updated value // is reflected on the datanodes. long newBandwidth = 12 * DEFAULT_BANDWIDTH; // 12M bps fs.setBalancerBandwidth(newBandwidth); + verifyBalancerBandwidth(datanodes, newBandwidth); - // Give it a few seconds to propogate new the value to the datanodes. - try { - Thread.sleep(5000); - } catch (Exception e) {} - - assertEquals(newBandwidth, (long) datanodes.get(0).getBalancerBandwidth()); - assertEquals(newBandwidth, (long) datanodes.get(1).getBalancerBandwidth()); // verifies the dfsadmin command execution args = new String[] { "-getBalancerBandwidth", dn1Address }; - runGetBalancerBandwidthCmd(admin, args, dn1Proxy, newBandwidth); + runGetBalancerBandwidthCmd(admin, args, newBandwidth); args = new String[] { "-getBalancerBandwidth", dn2Address }; - runGetBalancerBandwidthCmd(admin, args, dn2Proxy, newBandwidth); + runGetBalancerBandwidthCmd(admin, args, newBandwidth); // Dynamically change balancer bandwidth to 0. Balancer bandwidth on the // datanodes should remain as it was. fs.setBalancerBandwidth(0); - // Give it a few seconds to propogate new the value to the datanodes. - try { - Thread.sleep(5000); - } catch (Exception e) {} + verifyBalancerBandwidth(datanodes, newBandwidth); - assertEquals(newBandwidth, (long) datanodes.get(0).getBalancerBandwidth()); - assertEquals(newBandwidth, (long) datanodes.get(1).getBalancerBandwidth()); // verifies the dfsadmin command execution args = new String[] { "-getBalancerBandwidth", dn1Address }; - runGetBalancerBandwidthCmd(admin, args, dn1Proxy, newBandwidth); + runGetBalancerBandwidthCmd(admin, args, newBandwidth); args = new String[] { "-getBalancerBandwidth", dn2Address }; - runGetBalancerBandwidthCmd(admin, args, dn2Proxy, newBandwidth); - } finally { - cluster.shutdown(); + runGetBalancerBandwidthCmd(admin, args, newBandwidth); } } + private void verifyBalancerBandwidth(final ArrayList datanodes, + final long newBandwidth) throws TimeoutException, InterruptedException { + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return (long) datanodes.get(0).getBalancerBandwidth() == newBandwidth + && (long) datanodes.get(1).getBalancerBandwidth() == newBandwidth; + } + }, 100, 60 * 1000); + } + private void runGetBalancerBandwidthCmd(DFSAdmin admin, String[] args, - ClientDatanodeProtocol proxy, long expectedBandwidth) throws Exception { + long expectedBandwidth) throws Exception { PrintStream initialStdOut = System.out; outContent.reset(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index c7997d7d0fd..6db70d59d57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -103,7 +103,6 @@ import org.mockito.internal.stubbing.answers.ThrowsException; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.google.common.base.Joiner; /** * These tests make sure that DFSClient retries fetching data from DFS @@ -485,8 +484,7 @@ public class TestDFSClientRetries { // complete() may return false a few times before it returns // true. We want to wait until it returns true, and then // make it retry one more time after that. - LOG.info("Called complete(: " + - Joiner.on(",").join(invocation.getArguments()) + ")"); + LOG.info("Called complete:"); if (!(Boolean)invocation.callRealMethod()) { LOG.info("Complete call returned false, not faking a retry RPC"); return false; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index fc90db5acdf..5e5b8b6213c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -66,6 +66,10 @@ import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; +import org.junit.rules.Timeout; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS; @@ -95,6 +99,37 @@ public class TestDFSShell { private static final byte[] RAW_A1_VALUE = new byte[]{0x32, 0x32, 0x32}; private static final byte[] TRUSTED_A1_VALUE = new byte[]{0x31, 0x31, 0x31}; private static final byte[] USER_A1_VALUE = new byte[]{0x31, 0x32, 0x33}; + private static final int BLOCK_SIZE = 1024; + + private static MiniDFSCluster miniCluster; + private static DistributedFileSystem dfs; + + @BeforeClass + public static void setup() throws IOException { + final Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true); + conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + // set up the shared miniCluster directory so individual tests can launch + // new clusters without conflict + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, + GenericTestUtils.getTestDir("TestDFSShell").getAbsolutePath()); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); + + miniCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + miniCluster.waitActive(); + dfs = miniCluster.getFileSystem(); + } + + @AfterClass + public static void tearDown() { + if (miniCluster != null) { + miniCluster.shutdown(true, true); + } + } + + @Rule + public Timeout globalTimeout= new Timeout(30 * 1000); // 30s static Path writeFile(FileSystem fs, Path f) throws IOException { DataOutputStream out = fs.create(f); @@ -154,106 +189,77 @@ public class TestDFSShell { @Test (timeout = 30000) public void testZeroSizeFile() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = cluster.getFileSystem(); - assertTrue("Not a HDFS: "+fs.getUri(), - fs instanceof DistributedFileSystem); - final DistributedFileSystem dfs = (DistributedFileSystem)fs; + //create a zero size file + final File f1 = new File(TEST_ROOT_DIR, "f1"); + assertTrue(!f1.exists()); + assertTrue(f1.createNewFile()); + assertTrue(f1.exists()); + assertTrue(f1.isFile()); + assertEquals(0L, f1.length()); - try { - //create a zero size file - final File f1 = new File(TEST_ROOT_DIR, "f1"); - assertTrue(!f1.exists()); - assertTrue(f1.createNewFile()); - assertTrue(f1.exists()); - assertTrue(f1.isFile()); - assertEquals(0L, f1.length()); + //copy to remote + final Path root = mkdir(dfs, new Path("/testZeroSizeFile/zeroSizeFile")); + final Path remotef = new Path(root, "dst"); + show("copy local " + f1 + " to remote " + remotef); + dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), remotef); - //copy to remote - final Path root = mkdir(dfs, new Path("/test/zeroSizeFile")); - final Path remotef = new Path(root, "dst"); - show("copy local " + f1 + " to remote " + remotef); - dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), remotef); + //getBlockSize() should not throw exception + show("Block size = " + dfs.getFileStatus(remotef).getBlockSize()); - //getBlockSize() should not throw exception - show("Block size = " + dfs.getFileStatus(remotef).getBlockSize()); + //copy back + final File f2 = new File(TEST_ROOT_DIR, "f2"); + assertTrue(!f2.exists()); + dfs.copyToLocalFile(remotef, new Path(f2.getPath())); + assertTrue(f2.exists()); + assertTrue(f2.isFile()); + assertEquals(0L, f2.length()); - //copy back - final File f2 = new File(TEST_ROOT_DIR, "f2"); - assertTrue(!f2.exists()); - dfs.copyToLocalFile(remotef, new Path(f2.getPath())); - assertTrue(f2.exists()); - assertTrue(f2.isFile()); - assertEquals(0L, f2.length()); - - f1.delete(); - f2.delete(); - } finally { - try {dfs.close();} catch (Exception e) {} - cluster.shutdown(); - } + f1.delete(); + f2.delete(); } @Test (timeout = 30000) public void testRecursiveRm() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = cluster.getFileSystem(); - assertTrue("Not a HDFS: " + fs.getUri(), - fs instanceof DistributedFileSystem); - try { - fs.mkdirs(new Path(new Path("parent"), "child")); - try { - fs.delete(new Path("parent"), false); - assert(false); // should never reach here. - } catch(IOException e) { - //should have thrown an exception - } - try { - fs.delete(new Path("parent"), true); - } catch(IOException e) { - assert(false); - } - } finally { - try { fs.close();}catch(IOException e){}; - cluster.shutdown(); + final Path parent = new Path("/testRecursiveRm", "parent"); + final Path child = new Path(parent, "child"); + dfs.mkdirs(child); + try { + dfs.delete(parent, false); + fail("Should have failed because dir is not empty"); + } catch(IOException e) { + //should have thrown an exception } + dfs.delete(parent, true); + assertFalse(dfs.exists(parent)); } @Test (timeout = 30000) public void testDu() throws IOException { int replication = 2; - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(replication).build(); - DistributedFileSystem fs = cluster.getFileSystem(); PrintStream psBackup = System.out; ByteArrayOutputStream out = new ByteArrayOutputStream(); PrintStream psOut = new PrintStream(out); System.setOut(psOut); - FsShell shell = new FsShell(); - shell.setConf(conf); + FsShell shell = new FsShell(dfs.getConf()); try { - cluster.waitActive(); - Path myPath = new Path("/test/dir"); - assertTrue(fs.mkdirs(myPath)); - assertTrue(fs.exists(myPath)); - Path myFile = new Path("/test/dir/file"); - writeFile(fs, myFile); - assertTrue(fs.exists(myFile)); - Path myFile2 = new Path("/test/dir/file2"); - writeFile(fs, myFile2); - assertTrue(fs.exists(myFile2)); - Long myFileLength = fs.getFileStatus(myFile).getLen(); + final Path myPath = new Path("/testDu", "dir"); + assertTrue(dfs.mkdirs(myPath)); + assertTrue(dfs.exists(myPath)); + final Path myFile = new Path(myPath, "file"); + writeFile(dfs, myFile); + assertTrue(dfs.exists(myFile)); + final Path myFile2 = new Path(myPath, "file2"); + writeFile(dfs, myFile2); + assertTrue(dfs.exists(myFile2)); + Long myFileLength = dfs.getFileStatus(myFile).getLen(); Long myFileDiskUsed = myFileLength * replication; - Long myFile2Length = fs.getFileStatus(myFile2).getLen(); + Long myFile2Length = dfs.getFileStatus(myFile2).getLen(); Long myFile2DiskUsed = myFile2Length * replication; String[] args = new String[2]; args[0] = "-du"; - args[1] = "/test/dir"; + args[1] = myPath.toString(); int val = -1; try { val = shell.run(args); @@ -273,10 +279,10 @@ public class TestDFSShell { // Check that -du -s reports the state of the snapshot String snapshotName = "ss1"; Path snapshotPath = new Path(myPath, ".snapshot/" + snapshotName); - fs.allowSnapshot(myPath); - assertThat(fs.createSnapshot(myPath, snapshotName), is(snapshotPath)); - assertThat(fs.delete(myFile, false), is(true)); - assertThat(fs.exists(myFile), is(false)); + dfs.allowSnapshot(myPath); + assertThat(dfs.createSnapshot(myPath, snapshotName), is(snapshotPath)); + assertThat(dfs.delete(myFile, false), is(true)); + assertThat(dfs.exists(myFile), is(false)); args = new String[3]; args[0] = "-du"; @@ -298,13 +304,13 @@ public class TestDFSShell { assertThat(returnString, containsString(combinedDiskUsed.toString())); // Check if output is rendered properly with multiple input paths - Path myFile3 = new Path("/test/dir/file3"); - writeByte(fs, myFile3); - assertTrue(fs.exists(myFile3)); + final Path myFile3 = new Path(myPath, "file3"); + writeByte(dfs, myFile3); + assertTrue(dfs.exists(myFile3)); args = new String[3]; args[0] = "-du"; - args[1] = "/test/dir/file3"; - args[2] = "/test/dir/file2"; + args[1] = myFile3.toString(); + args[2] = myFile2.toString(); val = -1; try { val = shell.run(args); @@ -315,31 +321,24 @@ public class TestDFSShell { assertEquals("Return code should be 0.", 0, val); returnString = out.toString(); out.reset(); - assertTrue(returnString.contains("1 2 /test/dir/file3")); - assertTrue(returnString.contains("23 46 /test/dir/file2")); + assertTrue(returnString.contains("1 2 " + myFile3.toString())); + assertTrue(returnString.contains("25 50 " + myFile2.toString())); } finally { System.setOut(psBackup); - cluster.shutdown(); } } @Test (timeout = 180000) public void testDuSnapshots() throws IOException { final int replication = 2; - final Configuration conf = new HdfsConfiguration(); - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(replication).build(); - final DistributedFileSystem dfs = cluster.getFileSystem(); final PrintStream psBackup = System.out; final ByteArrayOutputStream out = new ByteArrayOutputStream(); final PrintStream psOut = new PrintStream(out); - final FsShell shell = new FsShell(); - shell.setConf(conf); + final FsShell shell = new FsShell(dfs.getConf()); try { System.setOut(psOut); - cluster.waitActive(); - final Path parent = new Path("/test"); + final Path parent = new Path("/testDuSnapshots"); final Path dir = new Path(parent, "dir"); mkdir(dfs, dir); final Path file = new Path(dir, "file"); @@ -453,27 +452,19 @@ public class TestDFSShell { out.reset(); } finally { System.setOut(psBackup); - cluster.shutdown(); } } @Test (timeout = 180000) public void testCountSnapshots() throws IOException { - final int replication = 2; - final Configuration conf = new HdfsConfiguration(); - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(replication).build(); - final DistributedFileSystem dfs = cluster.getFileSystem(); final PrintStream psBackup = System.out; final ByteArrayOutputStream out = new ByteArrayOutputStream(); final PrintStream psOut = new PrintStream(out); System.setOut(psOut); - final FsShell shell = new FsShell(); - shell.setConf(conf); + final FsShell shell = new FsShell(dfs.getConf()); try { - cluster.waitActive(); - final Path parent = new Path("/test"); + final Path parent = new Path("/testCountSnapshots"); final Path dir = new Path(parent, "dir"); mkdir(dfs, dir); final Path file = new Path(dir, "file"); @@ -545,117 +536,99 @@ public class TestDFSShell { out.reset(); } finally { System.setOut(psBackup); - cluster.shutdown(); } } @Test (timeout = 30000) public void testPut() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = cluster.getFileSystem(); - assertTrue("Not a HDFS: "+fs.getUri(), - fs instanceof DistributedFileSystem); - final DistributedFileSystem dfs = (DistributedFileSystem)fs; + // remove left over crc files: + new File(TEST_ROOT_DIR, ".f1.crc").delete(); + new File(TEST_ROOT_DIR, ".f2.crc").delete(); + final File f1 = createLocalFile(new File(TEST_ROOT_DIR, "f1")); + final File f2 = createLocalFile(new File(TEST_ROOT_DIR, "f2")); - try { - // remove left over crc files: - new File(TEST_ROOT_DIR, ".f1.crc").delete(); - new File(TEST_ROOT_DIR, ".f2.crc").delete(); - final File f1 = createLocalFile(new File(TEST_ROOT_DIR, "f1")); - final File f2 = createLocalFile(new File(TEST_ROOT_DIR, "f2")); + final Path root = mkdir(dfs, new Path("/testPut")); + final Path dst = new Path(root, "dst"); - final Path root = mkdir(dfs, new Path("/test/put")); - final Path dst = new Path(root, "dst"); + show("begin"); - show("begin"); - - final Thread copy2ndFileThread = new Thread() { - @Override - public void run() { - try { - show("copy local " + f2 + " to remote " + dst); - dfs.copyFromLocalFile(false, false, new Path(f2.getPath()), dst); - } catch (IOException ioe) { - show("good " + StringUtils.stringifyException(ioe)); - return; - } - //should not be here, must got IOException - assertTrue(false); + final Thread copy2ndFileThread = new Thread() { + @Override + public void run() { + try { + show("copy local " + f2 + " to remote " + dst); + dfs.copyFromLocalFile(false, false, new Path(f2.getPath()), dst); + } catch (IOException ioe) { + show("good " + StringUtils.stringifyException(ioe)); + return; } - }; + //should not be here, must got IOException + assertTrue(false); + } + }; - //use SecurityManager to pause the copying of f1 and begin copying f2 - SecurityManager sm = System.getSecurityManager(); - System.out.println("SecurityManager = " + sm); - System.setSecurityManager(new SecurityManager() { - private boolean firstTime = true; + //use SecurityManager to pause the copying of f1 and begin copying f2 + SecurityManager sm = System.getSecurityManager(); + System.out.println("SecurityManager = " + sm); + System.setSecurityManager(new SecurityManager() { + private boolean firstTime = true; - @Override - public void checkPermission(Permission perm) { - if (firstTime) { - Thread t = Thread.currentThread(); - if (!t.toString().contains("DataNode")) { - String s = "" + Arrays.asList(t.getStackTrace()); - if (s.contains("FileUtil.copyContent")) { - //pause at FileUtil.copyContent + @Override + public void checkPermission(Permission perm) { + if (firstTime) { + Thread t = Thread.currentThread(); + if (!t.toString().contains("DataNode")) { + String s = "" + Arrays.asList(t.getStackTrace()); + if (s.contains("FileUtil.copyContent")) { + //pause at FileUtil.copyContent - firstTime = false; - copy2ndFileThread.start(); - try {Thread.sleep(5000);} catch (InterruptedException e) {} - } + firstTime = false; + copy2ndFileThread.start(); + try {Thread.sleep(5000);} catch (InterruptedException e) {} } } } - }); - show("copy local " + f1 + " to remote " + dst); - dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), dst); - show("done"); + } + }); + show("copy local " + f1 + " to remote " + dst); + dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), dst); + show("done"); - try {copy2ndFileThread.join();} catch (InterruptedException e) { } - System.setSecurityManager(sm); + try {copy2ndFileThread.join();} catch (InterruptedException e) { } + System.setSecurityManager(sm); - // copy multiple files to destination directory - final Path destmultiple = mkdir(dfs, new Path("/test/putmultiple")); - Path[] srcs = new Path[2]; - srcs[0] = new Path(f1.getPath()); - srcs[1] = new Path(f2.getPath()); - dfs.copyFromLocalFile(false, false, srcs, destmultiple); - srcs[0] = new Path(destmultiple,"f1"); - srcs[1] = new Path(destmultiple,"f2"); - assertTrue(dfs.exists(srcs[0])); - assertTrue(dfs.exists(srcs[1])); + // copy multiple files to destination directory + final Path destmultiple = mkdir(dfs, new Path(root, "putmultiple")); + Path[] srcs = new Path[2]; + srcs[0] = new Path(f1.getPath()); + srcs[1] = new Path(f2.getPath()); + dfs.copyFromLocalFile(false, false, srcs, destmultiple); + srcs[0] = new Path(destmultiple,"f1"); + srcs[1] = new Path(destmultiple,"f2"); + assertTrue(dfs.exists(srcs[0])); + assertTrue(dfs.exists(srcs[1])); - // move multiple files to destination directory - final Path destmultiple2 = mkdir(dfs, new Path("/test/movemultiple")); - srcs[0] = new Path(f1.getPath()); - srcs[1] = new Path(f2.getPath()); - dfs.moveFromLocalFile(srcs, destmultiple2); - assertFalse(f1.exists()); - assertFalse(f2.exists()); - srcs[0] = new Path(destmultiple2, "f1"); - srcs[1] = new Path(destmultiple2, "f2"); - assertTrue(dfs.exists(srcs[0])); - assertTrue(dfs.exists(srcs[1])); + // move multiple files to destination directory + final Path destmultiple2 = mkdir(dfs, new Path(root, "movemultiple")); + srcs[0] = new Path(f1.getPath()); + srcs[1] = new Path(f2.getPath()); + dfs.moveFromLocalFile(srcs, destmultiple2); + assertFalse(f1.exists()); + assertFalse(f2.exists()); + srcs[0] = new Path(destmultiple2, "f1"); + srcs[1] = new Path(destmultiple2, "f2"); + assertTrue(dfs.exists(srcs[0])); + assertTrue(dfs.exists(srcs[1])); - f1.delete(); - f2.delete(); - } finally { - try {dfs.close();} catch (Exception e) {} - cluster.shutdown(); - } + f1.delete(); + f2.delete(); } - /** check command error outputs and exit statuses. */ @Test (timeout = 30000) public void testErrOutPut() throws Exception { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = null; PrintStream bak = null; try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem srcFs = cluster.getFileSystem(); Path root = new Path("/nonexistentfile"); bak = System.err; ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -672,8 +645,7 @@ public class TestDFSShell { out.reset(); argv[0] = "-rm"; argv[1] = root.toString(); - FsShell shell = new FsShell(); - shell.setConf(conf); + FsShell shell = new FsShell(dfs.getConf()); ret = ToolRunner.run(shell, argv); assertEquals(" -rm returned 1 ", 1, ret); returned = out.toString(); @@ -714,7 +686,7 @@ public class TestDFSShell { ret = ToolRunner.run(shell, argv); assertEquals(" -lsr should fail ", 1, ret); out.reset(); - srcFs.mkdirs(new Path("/testdir")); + dfs.mkdirs(new Path("/testdir")); argv[0] = "-ls"; argv[1] = "/testdir"; ret = ToolRunner.run(shell, argv); @@ -735,7 +707,7 @@ public class TestDFSShell { assertTrue(" -mkdir returned File exists", (returned.lastIndexOf("File exists") != -1)); Path testFile = new Path("/testfile"); - OutputStream outtmp = srcFs.create(testFile); + OutputStream outtmp = dfs.create(testFile); outtmp.write(testFile.toString().getBytes()); outtmp.close(); out.reset(); @@ -750,7 +722,7 @@ public class TestDFSShell { argv = new String[3]; argv[0] = "-mv"; argv[1] = "/testfile"; - argv[2] = "file"; + argv[2] = "/no-such-dir/file"; ret = ToolRunner.run(shell, argv); assertEquals("mv failed to rename", 1, ret); out.reset(); @@ -773,7 +745,7 @@ public class TestDFSShell { out.reset(); argv = new String[1]; argv[0] = "-du"; - srcFs.mkdirs(srcFs.getHomeDirectory()); + dfs.mkdirs(dfs.getHomeDirectory()); ret = ToolRunner.run(shell, argv); returned = out.toString(); assertEquals(" no error ", 0, ret); @@ -792,9 +764,6 @@ public class TestDFSShell { if (bak != null) { System.setErr(bak); } - if (cluster != null) { - cluster.shutdown(); - } } } @@ -836,7 +805,7 @@ public class TestDFSShell { Configuration dstConf = new HdfsConfiguration(); MiniDFSCluster srcCluster = null; MiniDFSCluster dstCluster = null; - File bak = new File(PathUtils.getTestDir(getClass()), "dfs_tmp_uri"); + File bak = new File(PathUtils.getTestDir(getClass()), "testURIPaths"); bak.mkdirs(); try{ srcCluster = new MiniDFSCluster.Builder(srcConf).numDataNodes(2).build(); @@ -926,35 +895,26 @@ public class TestDFSShell { */ @Test (timeout = 30000) public void testTail() throws Exception { - final int blockSize = 1024; - final int fileLen = 5 * blockSize; - final Configuration conf = new Configuration(); - conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + final int fileLen = 5 * BLOCK_SIZE; - try (MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(3).build()) { - cluster.waitActive(); - final DistributedFileSystem dfs = cluster.getFileSystem(); - - // create a text file with multiple KB bytes (and multiple blocks) - final Path testFile = new Path("testTail", "file1"); - final String text = RandomStringUtils.randomAscii(fileLen); - try (OutputStream pout = dfs.create(testFile)) { - pout.write(text.getBytes()); - } - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - System.setOut(new PrintStream(out)); - final String[] argv = new String[]{"-tail", testFile.toString()}; - final int ret = ToolRunner.run(new FsShell(conf), argv); - - assertEquals(Arrays.toString(argv) + " returned " + ret, 0, ret); - assertEquals("-tail returned " + out.size() + " bytes data, expected 1KB", - 1024, out.size()); - // tailed out last 1KB of the file content - assertArrayEquals("Tail output doesn't match input", - text.substring(fileLen - 1024).getBytes(), out.toByteArray()); - out.reset(); + // create a text file with multiple KB bytes (and multiple blocks) + final Path testFile = new Path("testTail", "file1"); + final String text = RandomStringUtils.randomAscii(fileLen); + try (OutputStream pout = dfs.create(testFile)) { + pout.write(text.getBytes()); } + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + final String[] argv = new String[]{"-tail", testFile.toString()}; + final int ret = ToolRunner.run(new FsShell(dfs.getConf()), argv); + + assertEquals(Arrays.toString(argv) + " returned " + ret, 0, ret); + assertEquals("-tail returned " + out.size() + " bytes data, expected 1KB", + 1024, out.size()); + // tailed out last 1KB of the file content + assertArrayEquals("Tail output doesn't match input", + text.substring(fileLen - 1024).getBytes(), out.toByteArray()); + out.reset(); } /** @@ -962,75 +922,56 @@ public class TestDFSShell { */ @Test(timeout = 30000) public void testTailWithFresh() throws Exception { - final int blockSize = 1024; - final Configuration conf = new Configuration(); - conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + final Path testFile = new Path("testTailWithFresh", "file1"); + dfs.create(testFile); - try (MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(3).build()) { - cluster.waitActive(); - final DistributedFileSystem dfs = cluster.getFileSystem(); - final Path testFile = new Path("testTailWithFresh", "file1"); - dfs.create(testFile); - - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - System.setOut(new PrintStream(out)); - final Thread tailer = new Thread() { - @Override - public void run() { - final String[] argv = new String[]{"-tail", "-f", - testFile.toString()}; - try { - ToolRunner.run(new FsShell(conf), argv); - } catch (Exception e) { - LOG.error("Client that tails the test file fails", e); - } finally { - out.reset(); - } + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + final Thread tailer = new Thread() { + @Override + public void run() { + final String[] argv = new String[]{"-tail", "-f", + testFile.toString()}; + try { + ToolRunner.run(new FsShell(dfs.getConf()), argv); + } catch (Exception e) { + LOG.error("Client that tails the test file fails", e); + } finally { + out.reset(); } - }; - tailer.start(); - // wait till the tailer is sleeping - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - return tailer.getState() == Thread.State.TIMED_WAITING; - } - }, 100, 10000); - - final String text = RandomStringUtils.randomAscii(blockSize / 2); - try (OutputStream pout = dfs.create(testFile)) { - pout.write(text.getBytes()); } - // The tailer should eventually show the file contents - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - return Arrays.equals(text.getBytes(), out.toByteArray()); - } - }, 100, 10000); + }; + tailer.start(); + // wait till the tailer is sleeping + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return tailer.getState() == Thread.State.TIMED_WAITING; + } + }, 100, 10000); + + final String text = RandomStringUtils.randomAscii(BLOCK_SIZE / 2); + try (OutputStream pout = dfs.create(testFile)) { + pout.write(text.getBytes()); } + // The tailer should eventually show the file contents + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return Arrays.equals(text.getBytes(), out.toByteArray()); + } + }, 100, 10000); } @Test (timeout = 30000) public void testText() throws Exception { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = null; - try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - final FileSystem dfs = cluster.getFileSystem(); - textTest(new Path("/texttest").makeQualified(dfs.getUri(), - dfs.getWorkingDirectory()), conf); + final Configuration conf = dfs.getConf(); + textTest(new Path("/texttest").makeQualified(dfs.getUri(), + dfs.getWorkingDirectory()), conf); - conf.set("fs.defaultFS", dfs.getUri().toString()); - final FileSystem lfs = FileSystem.getLocal(conf); - textTest(new Path(TEST_ROOT_DIR, "texttest").makeQualified(lfs.getUri(), - lfs.getWorkingDirectory()), conf); - } finally { - if (null != cluster) { - cluster.shutdown(); - } - } + final FileSystem lfs = FileSystem.getLocal(conf); + textTest(new Path(TEST_ROOT_DIR, "texttest").makeQualified(lfs.getUri(), + lfs.getWorkingDirectory()), conf); } private void textTest(Path root, Configuration conf) throws Exception { @@ -1145,75 +1086,60 @@ public class TestDFSShell { @Test (timeout = 30000) public void testCopyToLocal() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = cluster.getFileSystem(); - assertTrue("Not a HDFS: "+fs.getUri(), - fs instanceof DistributedFileSystem); - DistributedFileSystem dfs = (DistributedFileSystem)fs; - FsShell shell = new FsShell(); - shell.setConf(conf); + FsShell shell = new FsShell(dfs.getConf()); - try { - String root = createTree(dfs, "copyToLocal"); + String root = createTree(dfs, "copyToLocal"); - // Verify copying the tree - { - try { - assertEquals(0, - runCmd(shell, "-copyToLocal", root + "*", TEST_ROOT_DIR)); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - - File localroot = new File(TEST_ROOT_DIR, "copyToLocal"); - File localroot2 = new File(TEST_ROOT_DIR, "copyToLocal2"); - - File f1 = new File(localroot, "f1"); - assertTrue("Copying failed.", f1.isFile()); - - File f2 = new File(localroot, "f2"); - assertTrue("Copying failed.", f2.isFile()); - - File sub = new File(localroot, "sub"); - assertTrue("Copying failed.", sub.isDirectory()); - - File f3 = new File(sub, "f3"); - assertTrue("Copying failed.", f3.isFile()); - - File f4 = new File(sub, "f4"); - assertTrue("Copying failed.", f4.isFile()); - - File f5 = new File(localroot2, "f1"); - assertTrue("Copying failed.", f5.isFile()); - - f1.delete(); - f2.delete(); - f3.delete(); - f4.delete(); - f5.delete(); - sub.delete(); - } - // Verify copying non existing sources do not create zero byte - // destination files - { - String[] args = {"-copyToLocal", "nosuchfile", TEST_ROOT_DIR}; - try { - assertEquals(1, shell.run(args)); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - File f6 = new File(TEST_ROOT_DIR, "nosuchfile"); - assertTrue(!f6.exists()); - } - } finally { + // Verify copying the tree + { try { - dfs.close(); + assertEquals(0, + runCmd(shell, "-copyToLocal", root + "*", TEST_ROOT_DIR)); } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } - cluster.shutdown(); + + File localroot = new File(TEST_ROOT_DIR, "copyToLocal"); + File localroot2 = new File(TEST_ROOT_DIR, "copyToLocal2"); + + File f1 = new File(localroot, "f1"); + assertTrue("Copying failed.", f1.isFile()); + + File f2 = new File(localroot, "f2"); + assertTrue("Copying failed.", f2.isFile()); + + File sub = new File(localroot, "sub"); + assertTrue("Copying failed.", sub.isDirectory()); + + File f3 = new File(sub, "f3"); + assertTrue("Copying failed.", f3.isFile()); + + File f4 = new File(sub, "f4"); + assertTrue("Copying failed.", f4.isFile()); + + File f5 = new File(localroot2, "f1"); + assertTrue("Copying failed.", f5.isFile()); + + f1.delete(); + f2.delete(); + f3.delete(); + f4.delete(); + f5.delete(); + sub.delete(); + } + // Verify copying non existing sources do not create zero byte + // destination files + { + String[] args = {"-copyToLocal", "nosuchfile", TEST_ROOT_DIR}; + try { + assertEquals(1, shell.run(args)); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + File f6 = new File(TEST_ROOT_DIR, "nosuchfile"); + assertTrue(!f6.exists()); } } @@ -1243,63 +1169,42 @@ public class TestDFSShell { @Test (timeout = 30000) public void testCount() throws Exception { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - DistributedFileSystem dfs = cluster.getFileSystem(); - FsShell shell = new FsShell(); - shell.setConf(conf); + FsShell shell = new FsShell(dfs.getConf()); - try { - String root = createTree(dfs, "count"); + String root = createTree(dfs, "count"); - // Verify the counts - runCount(root, 2, 4, shell); - runCount(root + "2", 2, 1, shell); - runCount(root + "2/f1", 0, 1, shell); - runCount(root + "2/sub", 1, 0, shell); + // Verify the counts + runCount(root, 2, 4, shell); + runCount(root + "2", 2, 1, shell); + runCount(root + "2/f1", 0, 1, shell); + runCount(root + "2/sub", 1, 0, shell); - final FileSystem localfs = FileSystem.getLocal(conf); - Path localpath = new Path(TEST_ROOT_DIR, "testcount"); - localpath = localpath.makeQualified(localfs.getUri(), - localfs.getWorkingDirectory()); - localfs.mkdirs(localpath); + final FileSystem localfs = FileSystem.getLocal(dfs.getConf()); + Path localpath = new Path(TEST_ROOT_DIR, "testcount"); + localpath = localpath.makeQualified(localfs.getUri(), + localfs.getWorkingDirectory()); + localfs.mkdirs(localpath); - final String localstr = localpath.toString(); - System.out.println("localstr=" + localstr); - runCount(localstr, 1, 0, shell); - assertEquals(0, runCmd(shell, "-count", root, localstr)); - } finally { - try { - dfs.close(); - } catch (Exception e) { - } - cluster.shutdown(); - } + final String localstr = localpath.toString(); + System.out.println("localstr=" + localstr); + runCount(localstr, 1, 0, shell); + assertEquals(0, runCmd(shell, "-count", root, localstr)); } @Test(timeout = 30000) public void testTotalSizeOfAllFiles() throws Exception { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = null; - try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - FileSystem fs = cluster.getFileSystem(); - // create file under root - FSDataOutputStream File1 = fs.create(new Path("/File1")); - File1.write("hi".getBytes()); - File1.close(); - // create file under sub-folder - FSDataOutputStream File2 = fs.create(new Path("/Folder1/File2")); - File2.write("hi".getBytes()); - File2.close(); - // getUsed() should return total length of all the files in Filesystem - assertEquals(4, fs.getUsed()); - } finally { - if (cluster != null) { - cluster.shutdown(); - cluster = null; - } - } + final Path root = new Path("/testTotalSizeOfAllFiles"); + dfs.mkdirs(root); + // create file under root + FSDataOutputStream File1 = dfs.create(new Path(root, "File1")); + File1.write("hi".getBytes()); + File1.close(); + // create file under sub-folder + FSDataOutputStream File2 = dfs.create(new Path(root, "Folder1/File2")); + File2.write("hi".getBytes()); + File2.close(); + // getUsed() should return total length of all the files in Filesystem + assertEquals(4, dfs.getUsed(root)); } private static void runCount(String path, long dirs, long files, FsShell shell @@ -1414,7 +1319,6 @@ public class TestDFSShell { } finally { try { - fs.close(); shell.close(); } catch (IOException ignored) {} } @@ -1457,15 +1361,14 @@ public class TestDFSShell { conf.set(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, "true"); //test chmod on DFS - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - fs = cluster.getFileSystem(); + fs = dfs; + conf = dfs.getConf(); testChmod(conf, fs, "/tmp/chmodTest"); // test chown and chgrp on DFS: FsShell shell = new FsShell(); shell.setConf(conf); - fs = cluster.getFileSystem(); /* For dfs, I am the super user and I can change owner of any file to * anything. "-R" option is already tested by chmod test above. @@ -1505,439 +1408,423 @@ public class TestDFSShell { runCmd(shell, "-chgrp", "hadoop-core@apache.org/100", file); confirmOwner(null, "hadoop-core@apache.org/100", fs, path); - - cluster.shutdown(); } + /** * Tests various options of DFSShell. */ @Test (timeout = 120000) public void testDFSShell() throws Exception { - final Configuration conf = new HdfsConfiguration(); /* This tests some properties of ChecksumFileSystem as well. * Make sure that we create ChecksumDFS */ - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = cluster.getFileSystem(); - assertTrue("Not a HDFS: "+fs.getUri(), - fs instanceof DistributedFileSystem); - DistributedFileSystem fileSys = (DistributedFileSystem)fs; - FsShell shell = new FsShell(); - shell.setConf(conf); + FsShell shell = new FsShell(dfs.getConf()); - try { - // First create a new directory with mkdirs - Path myPath = new Path("/test/mkdirs"); - assertTrue(fileSys.mkdirs(myPath)); - assertTrue(fileSys.exists(myPath)); - assertTrue(fileSys.mkdirs(myPath)); + // First create a new directory with mkdirs + Path myPath = new Path("/testDFSShell/mkdirs"); + assertTrue(dfs.mkdirs(myPath)); + assertTrue(dfs.exists(myPath)); + assertTrue(dfs.mkdirs(myPath)); - // Second, create a file in that directory. - Path myFile = new Path("/test/mkdirs/myFile"); - writeFile(fileSys, myFile); - assertTrue(fileSys.exists(myFile)); - Path myFile2 = new Path("/test/mkdirs/myFile2"); - writeFile(fileSys, myFile2); - assertTrue(fileSys.exists(myFile2)); + // Second, create a file in that directory. + Path myFile = new Path("/testDFSShell/mkdirs/myFile"); + writeFile(dfs, myFile); + assertTrue(dfs.exists(myFile)); + Path myFile2 = new Path("/testDFSShell/mkdirs/myFile2"); + writeFile(dfs, myFile2); + assertTrue(dfs.exists(myFile2)); - // Verify that rm with a pattern - { - String[] args = new String[2]; - args[0] = "-rm"; - args[1] = "/test/mkdirs/myFile*"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertTrue(val == 0); - assertFalse(fileSys.exists(myFile)); - assertFalse(fileSys.exists(myFile2)); - - //re-create the files for other tests - writeFile(fileSys, myFile); - assertTrue(fileSys.exists(myFile)); - writeFile(fileSys, myFile2); - assertTrue(fileSys.exists(myFile2)); + // Verify that rm with a pattern + { + String[] args = new String[2]; + args[0] = "-rm"; + args[1] = "/testDFSShell/mkdirs/myFile*"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertTrue(val == 0); + assertFalse(dfs.exists(myFile)); + assertFalse(dfs.exists(myFile2)); - // Verify that we can read the file - { - String[] args = new String[3]; - args[0] = "-cat"; - args[1] = "/test/mkdirs/myFile"; - args[2] = "/test/mkdirs/myFile2"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run: " + - StringUtils.stringifyException(e)); - } - assertTrue(val == 0); + //re-create the files for other tests + writeFile(dfs, myFile); + assertTrue(dfs.exists(myFile)); + writeFile(dfs, myFile2); + assertTrue(dfs.exists(myFile2)); + } + + // Verify that we can read the file + { + String[] args = new String[3]; + args[0] = "-cat"; + args[1] = "/testDFSShell/mkdirs/myFile"; + args[2] = "/testDFSShell/mkdirs/myFile2"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run: " + + StringUtils.stringifyException(e)); } - fileSys.delete(myFile2, true); + assertTrue(val == 0); + } + dfs.delete(myFile2, true); - // Verify that we get an error while trying to read an nonexistent file - { - String[] args = new String[2]; - args[0] = "-cat"; - args[1] = "/test/mkdirs/myFile1"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertTrue(val != 0); + // Verify that we get an error while trying to read an nonexistent file + { + String[] args = new String[2]; + args[0] = "-cat"; + args[1] = "/testDFSShell/mkdirs/myFile1"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertTrue(val != 0); + } - // Verify that we get an error while trying to delete an nonexistent file - { - String[] args = new String[2]; - args[0] = "-rm"; - args[1] = "/test/mkdirs/myFile1"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertTrue(val != 0); + // Verify that we get an error while trying to delete an nonexistent file + { + String[] args = new String[2]; + args[0] = "-rm"; + args[1] = "/testDFSShell/mkdirs/myFile1"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertTrue(val != 0); + } - // Verify that we succeed in removing the file we created - { - String[] args = new String[2]; - args[0] = "-rm"; - args[1] = "/test/mkdirs/myFile"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertTrue(val == 0); + // Verify that we succeed in removing the file we created + { + String[] args = new String[2]; + args[0] = "-rm"; + args[1] = "/testDFSShell/mkdirs/myFile"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertTrue(val == 0); + } - // Verify touch/test - { - String[] args; - int val; + // Verify touch/test + { + String[] args; + int val; - args = new String[3]; - args[0] = "-test"; - args[1] = "-e"; - args[2] = "/test/mkdirs/noFileHere"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); - - args[1] = "-z"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); - - args = new String[2]; - args[0] = "-touchz"; - args[1] = "/test/mkdirs/isFileHere"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); - - args = new String[2]; - args[0] = "-touchz"; - args[1] = "/test/mkdirs/thisDirNotExists/isFileHere"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); - - - args = new String[3]; - args[0] = "-test"; - args[1] = "-e"; - args[2] = "/test/mkdirs/isFileHere"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); - - args[1] = "-d"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); - - args[1] = "-z"; - val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); + args = new String[3]; + args[0] = "-test"; + args[1] = "-e"; + args[2] = "/testDFSShell/mkdirs/noFileHere"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertEquals(1, val); - // Verify that cp from a directory to a subdirectory fails - { - String[] args = new String[2]; - args[0] = "-mkdir"; - args[1] = "/test/dir1"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); - - // this should fail - String[] args1 = new String[3]; - args1[0] = "-cp"; - args1[1] = "/test/dir1"; - args1[2] = "/test/dir1/dir2"; - val = 0; - try { - val = shell.run(args1); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); - - // this should succeed - args1[0] = "-cp"; - args1[1] = "/test/dir1"; - args1[2] = "/test/dir1foo"; - val = -1; - try { - val = shell.run(args1); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); - - // this should fail - args1[0] = "-cp"; - args1[1] = "/"; - args1[2] = "/test"; - val = 0; - try { - val = shell.run(args1); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); + args[1] = "-z"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertEquals(1, val); - // Verify -test -f negative case (missing file) - { - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-f"; - args[2] = "/test/mkdirs/noFileHere"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); + args = new String[2]; + args[0] = "-touchz"; + args[1] = "/testDFSShell/mkdirs/isFileHere"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertEquals(0, val); - // Verify -test -f negative case (directory rather than file) - { - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-f"; - args[2] = "/test/mkdirs"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); + args = new String[2]; + args[0] = "-touchz"; + args[1] = "/testDFSShell/mkdirs/thisDirNotExists/isFileHere"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertEquals(1, val); - // Verify -test -f positive case - { - writeFile(fileSys, myFile); - assertTrue(fileSys.exists(myFile)); - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-f"; - args[2] = myFile.toString(); - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); + args = new String[3]; + args[0] = "-test"; + args[1] = "-e"; + args[2] = "/testDFSShell/mkdirs/isFileHere"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertEquals(0, val); - // Verify -test -s negative case (missing file) - { - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-s"; - args[2] = "/test/mkdirs/noFileHere"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); + args[1] = "-d"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertEquals(1, val); - // Verify -test -s negative case (zero length file) - { - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-s"; - args[2] = "/test/mkdirs/isFileHere"; - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(1, val); + args[1] = "-z"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertEquals(0, val); + } - // Verify -test -s positive case (nonzero length file) - { - String[] args = new String[3]; - args[0] = "-test"; - args[1] = "-s"; - args[2] = myFile.toString(); - int val = -1; - try { - val = shell.run(args); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); + // Verify that cp from a directory to a subdirectory fails + { + String[] args = new String[2]; + args[0] = "-mkdir"; + args[1] = "/testDFSShell/dir1"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); } + assertEquals(0, val); - // Verify -test -w/-r - { - Path permDir = new Path("/test/permDir"); - Path permFile = new Path("/test/permDir/permFile"); - mkdir(fs, permDir); - writeFile(fs, permFile); + // this should fail + String[] args1 = new String[3]; + args1[0] = "-cp"; + args1[1] = "/testDFSShell/dir1"; + args1[2] = "/testDFSShell/dir1/dir2"; + val = 0; + try { + val = shell.run(args1); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); - // Verify -test -w positive case (dir exists and can write) - final String[] wargs = new String[3]; - wargs[0] = "-test"; - wargs[1] = "-w"; - wargs[2] = permDir.toString(); - int val = -1; - try { - val = shell.run(wargs); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); + // this should succeed + args1[0] = "-cp"; + args1[1] = "/testDFSShell/dir1"; + args1[2] = "/testDFSShell/dir1foo"; + val = -1; + try { + val = shell.run(args1); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); - // Verify -test -r positive case (file exists and can read) - final String[] rargs = new String[3]; - rargs[0] = "-test"; - rargs[1] = "-r"; - rargs[2] = permFile.toString(); - try { - val = shell.run(rargs); - } catch (Exception e) { - System.err.println("Exception raised from DFSShell.run " + - e.getLocalizedMessage()); - } - assertEquals(0, val); + // this should fail + args1[0] = "-cp"; + args1[1] = "/"; + args1[2] = "/test"; + val = 0; + try { + val = shell.run(args1); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); + } - // Verify -test -r negative case (file exists but cannot read) - runCmd(shell, "-chmod", "600", permFile.toString()); + // Verify -test -f negative case (missing file) + { + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-f"; + args[2] = "/testDFSShell/mkdirs/noFileHere"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); + } - UserGroupInformation smokeUser = - UserGroupInformation.createUserForTesting("smokeUser", - new String[] {"hadoop"}); - smokeUser.doAs(new PrivilegedExceptionAction() { - @Override - public String run() throws Exception { - FsShell shell = new FsShell(conf); - int exitCode = shell.run(rargs); - assertEquals(1, exitCode); - return null; - } - }); + // Verify -test -f negative case (directory rather than file) + { + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-f"; + args[2] = "/testDFSShell/mkdirs"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); + } - // Verify -test -w negative case (dir exists but cannot write) - runCmd(shell, "-chown", "-R", "not_allowed", permDir.toString()); - runCmd(shell, "-chmod", "-R", "700", permDir.toString()); + // Verify -test -f positive case + { + writeFile(dfs, myFile); + assertTrue(dfs.exists(myFile)); - smokeUser.doAs(new PrivilegedExceptionAction() { + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-f"; + args[2] = myFile.toString(); + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); + } + + // Verify -test -s negative case (missing file) + { + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-s"; + args[2] = "/testDFSShell/mkdirs/noFileHere"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); + } + + // Verify -test -s negative case (zero length file) + { + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-s"; + args[2] = "/testDFSShell/mkdirs/isFileHere"; + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); + } + + // Verify -test -s positive case (nonzero length file) + { + String[] args = new String[3]; + args[0] = "-test"; + args[1] = "-s"; + args[2] = myFile.toString(); + int val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); + } + + // Verify -test -w/-r + { + Path permDir = new Path("/testDFSShell/permDir"); + Path permFile = new Path("/testDFSShell/permDir/permFile"); + mkdir(dfs, permDir); + writeFile(dfs, permFile); + + // Verify -test -w positive case (dir exists and can write) + final String[] wargs = new String[3]; + wargs[0] = "-test"; + wargs[1] = "-w"; + wargs[2] = permDir.toString(); + int val = -1; + try { + val = shell.run(wargs); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); + + // Verify -test -r positive case (file exists and can read) + final String[] rargs = new String[3]; + rargs[0] = "-test"; + rargs[1] = "-r"; + rargs[2] = permFile.toString(); + try { + val = shell.run(rargs); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); + + // Verify -test -r negative case (file exists but cannot read) + runCmd(shell, "-chmod", "600", permFile.toString()); + + UserGroupInformation smokeUser = + UserGroupInformation.createUserForTesting("smokeUser", + new String[] {"hadoop"}); + smokeUser.doAs(new PrivilegedExceptionAction() { @Override public String run() throws Exception { - FsShell shell = new FsShell(conf); - int exitCode = shell.run(wargs); + FsShell shell = new FsShell(dfs.getConf()); + int exitCode = shell.run(rargs); assertEquals(1, exitCode); return null; } }); - // cleanup - fs.delete(permDir, true); - } - } finally { - try { - fileSys.close(); - } catch (Exception e) { - } - cluster.shutdown(); + // Verify -test -w negative case (dir exists but cannot write) + runCmd(shell, "-chown", "-R", "not_allowed", permDir.toString()); + runCmd(shell, "-chmod", "-R", "700", permDir.toString()); + + smokeUser.doAs(new PrivilegedExceptionAction() { + @Override + public String run() throws Exception { + FsShell shell = new FsShell(dfs.getConf()); + int exitCode = shell.run(wargs); + assertEquals(1, exitCode); + return null; + } + }); + + // cleanup + dfs.delete(permDir, true); } } @@ -1977,21 +1864,17 @@ public class TestDFSShell { public void testRemoteException() throws Exception { UserGroupInformation tmpUGI = UserGroupInformation.createUserForTesting("tmpname", new String[] {"mygroup"}); - MiniDFSCluster dfs = null; PrintStream bak = null; try { - final Configuration conf = new HdfsConfiguration(); - dfs = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = dfs.getFileSystem(); Path p = new Path("/foo"); - fs.mkdirs(p); - fs.setPermission(p, new FsPermission((short)0700)); + dfs.mkdirs(p); + dfs.setPermission(p, new FsPermission((short)0700)); bak = System.err; tmpUGI.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { - FsShell fshell = new FsShell(conf); + FsShell fshell = new FsShell(dfs.getConf()); ByteArrayOutputStream out = new ByteArrayOutputStream(); PrintStream tmp = new PrintStream(out); System.setErr(tmp); @@ -2011,9 +1894,6 @@ public class TestDFSShell { if (bak != null) { System.setErr(bak); } - if (dfs != null) { - dfs.shutdown(); - } } } @@ -2071,13 +1951,13 @@ public class TestDFSShell { // find block files to modify later List replicas = getMaterializedReplicas(cluster); - // Shut down cluster and then corrupt the block files by overwriting a - // portion with junk data. We must shut down the cluster so that threads + // Shut down miniCluster and then corrupt the block files by overwriting a + // portion with junk data. We must shut down the miniCluster so that threads // in the data node do not hold locks on the block files while we try to // write into them. Particularly on Windows, the data node's use of the // FileChannel.transferTo method can cause block files to be memory mapped // in read-only mode during the transfer to a client, and this causes a - // locking conflict. The call to shutdown the cluster blocks until all + // locking conflict. The call to shutdown the miniCluster blocks until all // DataXceiver threads exit, preventing this problem. dfs.close(); cluster.shutdown(); @@ -2085,7 +1965,7 @@ public class TestDFSShell { show("replicas=" + replicas); corrupt(replicas, localfcontent); - // Start the cluster again, but do not reformat, so prior files remain. + // Start the miniCluster again, but do not reformat, so prior files remain. cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(false) .build(); dfs = cluster.getFileSystem(); @@ -2114,55 +1994,45 @@ public class TestDFSShell { */ @Test (timeout = 30000) public void testStat() throws Exception { - final int blockSize = 1024; - final Configuration conf = new HdfsConfiguration(); - conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + final SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + fmt.setTimeZone(TimeZone.getTimeZone("UTC")); + final Path testDir1 = new Path("testStat", "dir1"); + dfs.mkdirs(testDir1); + final Path testFile2 = new Path(testDir1, "file2"); + DFSTestUtil.createFile(dfs, testFile2, 2 * BLOCK_SIZE, (short) 3, 0); + final FileStatus status1 = dfs.getFileStatus(testDir1); + final String mtime1 = fmt.format(new Date(status1.getModificationTime())); + final FileStatus status2 = dfs.getFileStatus(testFile2); + final String mtime2 = fmt.format(new Date(status2.getModificationTime())); - try (MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(3).build()) { - cluster.waitActive(); - final DistributedFileSystem dfs = cluster.getFileSystem(); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); - final SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - fmt.setTimeZone(TimeZone.getTimeZone("UTC")); - final Path testDir1 = new Path("testStat", "dir1"); - dfs.mkdirs(testDir1); - final FileStatus status1 = dfs.getFileStatus(testDir1); - final String mtime1 = fmt.format(new Date(status1.getModificationTime())); - final Path testFile2 = new Path(testDir1, "file2"); - DFSTestUtil.createFile(dfs, testFile2, 2 * blockSize, (short) 3, 0); - final FileStatus status2 = dfs.getFileStatus(testDir1); - final String mtime2 = fmt.format(new Date(status2.getModificationTime())); + doFsStat(dfs.getConf(), null); - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - System.setOut(new PrintStream(out)); + out.reset(); + doFsStat(dfs.getConf(), null, testDir1); + assertEquals("Unexpected -stat output: " + out, + out.toString(), String.format("%s%n", mtime1)); - doFsStat(conf, null); + out.reset(); + doFsStat(dfs.getConf(), null, testDir1, testFile2); + assertEquals("Unexpected -stat output: " + out, + out.toString(), String.format("%s%n%s%n", mtime1, mtime2)); - out.reset(); - doFsStat(conf, null, testDir1); - assertEquals("Unexpected -stat output: " + out, - out.toString(), String.format("%s%n", mtime1)); + doFsStat(dfs.getConf(), "%F %u:%g %b %y %n"); - out.reset(); - doFsStat(conf, null, testDir1, testFile2); - assertEquals("Unexpected -stat output: " + out, - out.toString(), String.format("%s%n%s%n", mtime1, mtime2)); + out.reset(); + doFsStat(dfs.getConf(), "%F %u:%g %b %y %n", testDir1); + assertTrue(out.toString(), out.toString().contains(mtime1)); + assertTrue(out.toString(), out.toString().contains("directory")); + assertTrue(out.toString(), out.toString().contains(status1.getGroup())); - doFsStat(conf, "%F %u:%g %b %y %n"); - - out.reset(); - doFsStat(conf, "%F %u:%g %b %y %n", testDir1); - assertTrue(out.toString(), out.toString().contains(mtime1)); - assertTrue(out.toString(), out.toString().contains("directory")); - assertTrue(out.toString(), out.toString().contains(status1.getGroup())); - - out.reset(); - doFsStat(conf, "%F %u:%g %b %y %n", testDir1, testFile2); - assertTrue(out.toString(), out.toString().contains(mtime1)); - assertTrue(out.toString(), out.toString().contains("regular file")); - assertTrue(out.toString(), out.toString().contains(mtime2)); - } + out.reset(); + doFsStat(dfs.getConf(), "%F %u:%g %b %y %n", testDir1, testFile2); + assertTrue(out.toString(), out.toString().contains(mtime1)); + assertTrue(out.toString(), out.toString().contains("regular file")); + assertTrue(out.toString(), out.toString().contains(mtime2)); } private static void doFsStat(Configuration conf, String format, Path... files) @@ -2189,33 +2059,26 @@ public class TestDFSShell { @Test (timeout = 30000) public void testLsr() throws Exception { - final Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - DistributedFileSystem dfs = cluster.getFileSystem(); + final Configuration conf = dfs.getConf(); + final String root = createTree(dfs, "lsr"); + dfs.mkdirs(new Path(root, "zzz")); - try { - final String root = createTree(dfs, "lsr"); - dfs.mkdirs(new Path(root, "zzz")); + runLsr(new FsShell(conf), root, 0); - runLsr(new FsShell(conf), root, 0); + final Path sub = new Path(root, "sub"); + dfs.setPermission(sub, new FsPermission((short)0)); - final Path sub = new Path(root, "sub"); - dfs.setPermission(sub, new FsPermission((short)0)); - - final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - final String tmpusername = ugi.getShortUserName() + "1"; - UserGroupInformation tmpUGI = UserGroupInformation.createUserForTesting( - tmpusername, new String[] {tmpusername}); - String results = tmpUGI.doAs(new PrivilegedExceptionAction() { - @Override - public String run() throws Exception { - return runLsr(new FsShell(conf), root, 1); - } - }); - assertTrue(results.contains("zzz")); - } finally { - cluster.shutdown(); - } + final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + final String tmpusername = ugi.getShortUserName() + "1"; + UserGroupInformation tmpUGI = UserGroupInformation.createUserForTesting( + tmpusername, new String[] {tmpusername}); + String results = tmpUGI.doAs(new PrivilegedExceptionAction() { + @Override + public String run() throws Exception { + return runLsr(new FsShell(conf), root, 1); + } + }); + assertTrue(results.contains("zzz")); } private static String runLsr(final FsShell shell, String root, int returnvalue @@ -2259,40 +2122,33 @@ public class TestDFSShell { // ACLs) @Test (timeout = 120000) public void testCopyCommandsWithPreserveOption() throws Exception { - Configuration conf = new Configuration(); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) - .format(true).build(); FsShell shell = null; - FileSystem fs = null; final String testdir = "/tmp/TestDFSShell-testCopyCommandsWithPreserveOption-" + counter.getAndIncrement(); final Path hdfsTestDir = new Path(testdir); try { - fs = cluster.getFileSystem(); - fs.mkdirs(hdfsTestDir); + dfs.mkdirs(hdfsTestDir); Path src = new Path(hdfsTestDir, "srcfile"); - fs.create(src).close(); + dfs.create(src).close(); - fs.setAcl(src, Lists.newArrayList( + dfs.setAcl(src, Lists.newArrayList( aclEntry(ACCESS, USER, ALL), aclEntry(ACCESS, USER, "foo", ALL), aclEntry(ACCESS, GROUP, READ_EXECUTE), aclEntry(ACCESS, GROUP, "bar", READ_EXECUTE), aclEntry(ACCESS, OTHER, EXECUTE))); - FileStatus status = fs.getFileStatus(src); + FileStatus status = dfs.getFileStatus(src); final long mtime = status.getModificationTime(); final long atime = status.getAccessTime(); final String owner = status.getOwner(); final String group = status.getGroup(); final FsPermission perm = status.getPermission(); - fs.setXAttr(src, USER_A1, USER_A1_VALUE); - fs.setXAttr(src, TRUSTED_A1, TRUSTED_A1_VALUE); + dfs.setXAttr(src, USER_A1, USER_A1_VALUE); + dfs.setXAttr(src, TRUSTED_A1, TRUSTED_A1_VALUE); - shell = new FsShell(conf); + shell = new FsShell(dfs.getConf()); // -p Path target1 = new Path(hdfsTestDir, "targetfile1"); @@ -2300,16 +2156,16 @@ public class TestDFSShell { target1.toUri().toString() }; int ret = ToolRunner.run(shell, argv); assertEquals("cp -p is not working", SUCCESS, ret); - FileStatus targetStatus = fs.getFileStatus(target1); + FileStatus targetStatus = dfs.getFileStatus(target1); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); FsPermission targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - Map xattrs = fs.getXAttrs(target1); + Map xattrs = dfs.getXAttrs(target1); assertTrue(xattrs.isEmpty()); - List acls = fs.getAclStatus(target1).getEntries(); + List acls = dfs.getAclStatus(target1).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2319,16 +2175,16 @@ public class TestDFSShell { target2.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptop is not working", SUCCESS, ret); - targetStatus = fs.getFileStatus(target2); + targetStatus = dfs.getFileStatus(target2); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = fs.getXAttrs(target2); + xattrs = dfs.getXAttrs(target2); assertTrue(xattrs.isEmpty()); - acls = fs.getAclStatus(target2).getEntries(); + acls = dfs.getAclStatus(target2).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2338,18 +2194,18 @@ public class TestDFSShell { target3.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptopx is not working", SUCCESS, ret); - targetStatus = fs.getFileStatus(target3); + targetStatus = dfs.getFileStatus(target3); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = fs.getXAttrs(target3); + xattrs = dfs.getXAttrs(target3); assertEquals(xattrs.size(), 2); assertArrayEquals(USER_A1_VALUE, xattrs.get(USER_A1)); assertArrayEquals(TRUSTED_A1_VALUE, xattrs.get(TRUSTED_A1)); - acls = fs.getAclStatus(target3).getEntries(); + acls = dfs.getAclStatus(target3).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2359,19 +2215,19 @@ public class TestDFSShell { target4.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptopa is not working", SUCCESS, ret); - targetStatus = fs.getFileStatus(target4); + targetStatus = dfs.getFileStatus(target4); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = fs.getXAttrs(target4); + xattrs = dfs.getXAttrs(target4); assertTrue(xattrs.isEmpty()); - acls = fs.getAclStatus(target4).getEntries(); + acls = dfs.getAclStatus(target4).getEntries(); assertFalse(acls.isEmpty()); assertTrue(targetPerm.getAclBit()); - assertEquals(fs.getAclStatus(src), fs.getAclStatus(target4)); + assertEquals(dfs.getAclStatus(src), dfs.getAclStatus(target4)); // -ptoa (verify -pa option will preserve permissions also) Path target5 = new Path(hdfsTestDir, "targetfile5"); @@ -2379,59 +2235,47 @@ public class TestDFSShell { target5.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptoa is not working", SUCCESS, ret); - targetStatus = fs.getFileStatus(target5); + targetStatus = dfs.getFileStatus(target5); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = fs.getXAttrs(target5); + xattrs = dfs.getXAttrs(target5); assertTrue(xattrs.isEmpty()); - acls = fs.getAclStatus(target5).getEntries(); + acls = dfs.getAclStatus(target5).getEntries(); assertFalse(acls.isEmpty()); assertTrue(targetPerm.getAclBit()); - assertEquals(fs.getAclStatus(src), fs.getAclStatus(target5)); + assertEquals(dfs.getAclStatus(src), dfs.getAclStatus(target5)); } finally { if (null != shell) { shell.close(); } - - if (null != fs) { - fs.delete(hdfsTestDir, true); - fs.close(); - } - cluster.shutdown(); } } @Test (timeout = 120000) public void testCopyCommandsWithRawXAttrs() throws Exception { - final Configuration conf = new Configuration(); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf). - numDataNodes(1).format(true).build(); FsShell shell = null; - FileSystem fs = null; final String testdir = "/tmp/TestDFSShell-testCopyCommandsWithRawXAttrs-" + counter.getAndIncrement(); final Path hdfsTestDir = new Path(testdir); final Path rawHdfsTestDir = new Path("/.reserved/raw" + testdir); try { - fs = cluster.getFileSystem(); - fs.mkdirs(hdfsTestDir); + dfs.mkdirs(hdfsTestDir); final Path src = new Path(hdfsTestDir, "srcfile"); final String rawSrcBase = "/.reserved/raw" + testdir; final Path rawSrc = new Path(rawSrcBase, "srcfile"); - fs.create(src).close(); + dfs.create(src).close(); final Path srcDir = new Path(hdfsTestDir, "srcdir"); final Path rawSrcDir = new Path("/.reserved/raw" + testdir, "srcdir"); - fs.mkdirs(srcDir); + dfs.mkdirs(srcDir); final Path srcDirFile = new Path(srcDir, "srcfile"); final Path rawSrcDirFile = new Path("/.reserved/raw" + srcDirFile); - fs.create(srcDirFile).close(); + dfs.create(srcDirFile).close(); final Path[] paths = { rawSrc, rawSrcDir, rawSrcDirFile }; final String[] xattrNames = { USER_A1, RAW_A1 }; @@ -2439,35 +2283,35 @@ public class TestDFSShell { for (int i = 0; i < paths.length; i++) { for (int j = 0; j < xattrNames.length; j++) { - fs.setXAttr(paths[i], xattrNames[j], xattrVals[j]); + dfs.setXAttr(paths[i], xattrNames[j], xattrVals[j]); } } - shell = new FsShell(conf); + shell = new FsShell(dfs.getConf()); /* Check that a file as the source path works ok. */ - doTestCopyCommandsWithRawXAttrs(shell, fs, src, hdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrc, hdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, fs, src, rawHdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrc, rawHdfsTestDir, true); + doTestCopyCommandsWithRawXAttrs(shell, dfs, src, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, dfs, rawSrc, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, dfs, src, rawHdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, dfs, rawSrc, rawHdfsTestDir, true); /* Use a relative /.reserved/raw path. */ - final Path savedWd = fs.getWorkingDirectory(); + final Path savedWd = dfs.getWorkingDirectory(); try { - fs.setWorkingDirectory(new Path(rawSrcBase)); + dfs.setWorkingDirectory(new Path(rawSrcBase)); final Path relRawSrc = new Path("../srcfile"); final Path relRawHdfsTestDir = new Path(".."); - doTestCopyCommandsWithRawXAttrs(shell, fs, relRawSrc, relRawHdfsTestDir, - true); + doTestCopyCommandsWithRawXAttrs(shell, dfs, relRawSrc, + relRawHdfsTestDir, true); } finally { - fs.setWorkingDirectory(savedWd); + dfs.setWorkingDirectory(savedWd); } /* Check that a directory as the source path works ok. */ - doTestCopyCommandsWithRawXAttrs(shell, fs, srcDir, hdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrcDir, hdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, fs, srcDir, rawHdfsTestDir, false); - doTestCopyCommandsWithRawXAttrs(shell, fs, rawSrcDir, rawHdfsTestDir, + doTestCopyCommandsWithRawXAttrs(shell, dfs, srcDir, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, dfs, rawSrcDir, hdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, dfs, srcDir, rawHdfsTestDir, false); + doTestCopyCommandsWithRawXAttrs(shell, dfs, rawSrcDir, rawHdfsTestDir, true); /* Use relative in an absolute path. */ @@ -2475,18 +2319,13 @@ public class TestDFSShell { testdir + "/srcdir"; final String relRawDstDir = "./.reserved/../.reserved/raw/../raw" + testdir; - doTestCopyCommandsWithRawXAttrs(shell, fs, new Path(relRawSrcDir), + doTestCopyCommandsWithRawXAttrs(shell, dfs, new Path(relRawSrcDir), new Path(relRawDstDir), true); } finally { if (null != shell) { shell.close(); } - - if (null != fs) { - fs.delete(hdfsTestDir, true); - fs.close(); - } - cluster.shutdown(); + dfs.delete(hdfsTestDir, true); } } @@ -2563,31 +2402,24 @@ public class TestDFSShell { @Test (timeout = 120000) public void testCopyCommandsToDirectoryWithPreserveOption() throws Exception { - Configuration conf = new Configuration(); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) - .format(true).build(); FsShell shell = null; - FileSystem fs = null; final String testdir = "/tmp/TestDFSShell-testCopyCommandsToDirectoryWithPreserveOption-" + counter.getAndIncrement(); final Path hdfsTestDir = new Path(testdir); try { - fs = cluster.getFileSystem(); - fs.mkdirs(hdfsTestDir); + dfs.mkdirs(hdfsTestDir); Path srcDir = new Path(hdfsTestDir, "srcDir"); - fs.mkdirs(srcDir); + dfs.mkdirs(srcDir); - fs.setAcl(srcDir, Lists.newArrayList( + dfs.setAcl(srcDir, Lists.newArrayList( aclEntry(ACCESS, USER, ALL), aclEntry(ACCESS, USER, "foo", ALL), aclEntry(ACCESS, GROUP, READ_EXECUTE), aclEntry(DEFAULT, GROUP, "bar", READ_EXECUTE), aclEntry(ACCESS, OTHER, EXECUTE))); // set sticky bit - fs.setPermission(srcDir, + dfs.setPermission(srcDir, new FsPermission(ALL, READ_EXECUTE, EXECUTE, true)); // Create a file in srcDir to check if modification time of @@ -2595,19 +2427,19 @@ public class TestDFSShell { // If cp -p command is to preserve modification time and then copy child // (srcFile), modification time will not be preserved. Path srcFile = new Path(srcDir, "srcFile"); - fs.create(srcFile).close(); + dfs.create(srcFile).close(); - FileStatus status = fs.getFileStatus(srcDir); + FileStatus status = dfs.getFileStatus(srcDir); final long mtime = status.getModificationTime(); final long atime = status.getAccessTime(); final String owner = status.getOwner(); final String group = status.getGroup(); final FsPermission perm = status.getPermission(); - fs.setXAttr(srcDir, USER_A1, USER_A1_VALUE); - fs.setXAttr(srcDir, TRUSTED_A1, TRUSTED_A1_VALUE); + dfs.setXAttr(srcDir, USER_A1, USER_A1_VALUE); + dfs.setXAttr(srcDir, TRUSTED_A1, TRUSTED_A1_VALUE); - shell = new FsShell(conf); + shell = new FsShell(dfs.getConf()); // -p Path targetDir1 = new Path(hdfsTestDir, "targetDir1"); @@ -2615,16 +2447,16 @@ public class TestDFSShell { targetDir1.toUri().toString() }; int ret = ToolRunner.run(shell, argv); assertEquals("cp -p is not working", SUCCESS, ret); - FileStatus targetStatus = fs.getFileStatus(targetDir1); + FileStatus targetStatus = dfs.getFileStatus(targetDir1); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); FsPermission targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - Map xattrs = fs.getXAttrs(targetDir1); + Map xattrs = dfs.getXAttrs(targetDir1); assertTrue(xattrs.isEmpty()); - List acls = fs.getAclStatus(targetDir1).getEntries(); + List acls = dfs.getAclStatus(targetDir1).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2634,16 +2466,16 @@ public class TestDFSShell { targetDir2.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptop is not working", SUCCESS, ret); - targetStatus = fs.getFileStatus(targetDir2); + targetStatus = dfs.getFileStatus(targetDir2); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = fs.getXAttrs(targetDir2); + xattrs = dfs.getXAttrs(targetDir2); assertTrue(xattrs.isEmpty()); - acls = fs.getAclStatus(targetDir2).getEntries(); + acls = dfs.getAclStatus(targetDir2).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2653,18 +2485,18 @@ public class TestDFSShell { targetDir3.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptopx is not working", SUCCESS, ret); - targetStatus = fs.getFileStatus(targetDir3); + targetStatus = dfs.getFileStatus(targetDir3); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = fs.getXAttrs(targetDir3); + xattrs = dfs.getXAttrs(targetDir3); assertEquals(xattrs.size(), 2); assertArrayEquals(USER_A1_VALUE, xattrs.get(USER_A1)); assertArrayEquals(TRUSTED_A1_VALUE, xattrs.get(TRUSTED_A1)); - acls = fs.getAclStatus(targetDir3).getEntries(); + acls = dfs.getAclStatus(targetDir3).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2674,19 +2506,19 @@ public class TestDFSShell { targetDir4.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptopa is not working", SUCCESS, ret); - targetStatus = fs.getFileStatus(targetDir4); + targetStatus = dfs.getFileStatus(targetDir4); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = fs.getXAttrs(targetDir4); + xattrs = dfs.getXAttrs(targetDir4); assertTrue(xattrs.isEmpty()); - acls = fs.getAclStatus(targetDir4).getEntries(); + acls = dfs.getAclStatus(targetDir4).getEntries(); assertFalse(acls.isEmpty()); assertTrue(targetPerm.getAclBit()); - assertEquals(fs.getAclStatus(srcDir), fs.getAclStatus(targetDir4)); + assertEquals(dfs.getAclStatus(srcDir), dfs.getAclStatus(targetDir4)); // -ptoa (verify -pa option will preserve permissions also) Path targetDir5 = new Path(hdfsTestDir, "targetDir5"); @@ -2694,68 +2526,57 @@ public class TestDFSShell { targetDir5.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptoa is not working", SUCCESS, ret); - targetStatus = fs.getFileStatus(targetDir5); + targetStatus = dfs.getFileStatus(targetDir5); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - xattrs = fs.getXAttrs(targetDir5); + xattrs = dfs.getXAttrs(targetDir5); assertTrue(xattrs.isEmpty()); - acls = fs.getAclStatus(targetDir5).getEntries(); + acls = dfs.getAclStatus(targetDir5).getEntries(); assertFalse(acls.isEmpty()); assertTrue(targetPerm.getAclBit()); - assertEquals(fs.getAclStatus(srcDir), fs.getAclStatus(targetDir5)); + assertEquals(dfs.getAclStatus(srcDir), dfs.getAclStatus(targetDir5)); } finally { if (shell != null) { shell.close(); } - if (fs != null) { - fs.delete(hdfsTestDir, true); - fs.close(); - } - cluster.shutdown(); } } // Verify cp -pa option will preserve both ACL and sticky bit. @Test (timeout = 120000) public void testCopyCommandsPreserveAclAndStickyBit() throws Exception { - Configuration conf = new Configuration(); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) - .format(true).build(); FsShell shell = null; - FileSystem fs = null; final String testdir = "/tmp/TestDFSShell-testCopyCommandsPreserveAclAndStickyBit-" + counter.getAndIncrement(); final Path hdfsTestDir = new Path(testdir); try { - fs = cluster.getFileSystem(); - fs.mkdirs(hdfsTestDir); + dfs.mkdirs(hdfsTestDir); Path src = new Path(hdfsTestDir, "srcfile"); - fs.create(src).close(); + dfs.create(src).close(); - fs.setAcl(src, Lists.newArrayList( + dfs.setAcl(src, Lists.newArrayList( aclEntry(ACCESS, USER, ALL), aclEntry(ACCESS, USER, "foo", ALL), aclEntry(ACCESS, GROUP, READ_EXECUTE), aclEntry(ACCESS, GROUP, "bar", READ_EXECUTE), aclEntry(ACCESS, OTHER, EXECUTE))); // set sticky bit - fs.setPermission(src, + dfs.setPermission(src, new FsPermission(ALL, READ_EXECUTE, EXECUTE, true)); - FileStatus status = fs.getFileStatus(src); + FileStatus status = dfs.getFileStatus(src); final long mtime = status.getModificationTime(); final long atime = status.getAccessTime(); final String owner = status.getOwner(); final String group = status.getGroup(); final FsPermission perm = status.getPermission(); - shell = new FsShell(conf); + shell = new FsShell(dfs.getConf()); // -p preserves sticky bit and doesn't preserve ACL Path target1 = new Path(hdfsTestDir, "targetfile1"); @@ -2763,14 +2584,14 @@ public class TestDFSShell { target1.toUri().toString() }; int ret = ToolRunner.run(shell, argv); assertEquals("cp is not working", SUCCESS, ret); - FileStatus targetStatus = fs.getFileStatus(target1); + FileStatus targetStatus = dfs.getFileStatus(target1); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); FsPermission targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - List acls = fs.getAclStatus(target1).getEntries(); + List acls = dfs.getAclStatus(target1).getEntries(); assertTrue(acls.isEmpty()); assertFalse(targetPerm.getAclBit()); @@ -2780,47 +2601,37 @@ public class TestDFSShell { target2.toUri().toString() }; ret = ToolRunner.run(shell, argv); assertEquals("cp -ptopa is not working", SUCCESS, ret); - targetStatus = fs.getFileStatus(target2); + targetStatus = dfs.getFileStatus(target2); assertEquals(mtime, targetStatus.getModificationTime()); assertEquals(atime, targetStatus.getAccessTime()); assertEquals(owner, targetStatus.getOwner()); assertEquals(group, targetStatus.getGroup()); targetPerm = targetStatus.getPermission(); assertTrue(perm.equals(targetPerm)); - acls = fs.getAclStatus(target2).getEntries(); + acls = dfs.getAclStatus(target2).getEntries(); assertFalse(acls.isEmpty()); assertTrue(targetPerm.getAclBit()); - assertEquals(fs.getAclStatus(src), fs.getAclStatus(target2)); + assertEquals(dfs.getAclStatus(src), dfs.getAclStatus(target2)); } finally { if (null != shell) { shell.close(); } - if (null != fs) { - fs.delete(hdfsTestDir, true); - fs.close(); - } - cluster.shutdown(); } } // force Copy Option is -f @Test (timeout = 30000) public void testCopyCommandsWithForceOption() throws Exception { - Configuration conf = new Configuration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) - .format(true).build(); FsShell shell = null; - FileSystem fs = null; final File localFile = new File(TEST_ROOT_DIR, "testFileForPut"); final String localfilepath = new Path(localFile.getAbsolutePath()).toUri().toString(); final String testdir = "/tmp/TestDFSShell-testCopyCommandsWithForceOption-" + counter.getAndIncrement(); final Path hdfsTestDir = new Path(testdir); try { - fs = cluster.getFileSystem(); - fs.mkdirs(hdfsTestDir); + dfs.mkdirs(hdfsTestDir); localFile.createNewFile(); - writeFile(fs, new Path(testdir, "testFileForPut")); + writeFile(dfs, new Path(testdir, "testFileForPut")); shell = new FsShell(); // Tests for put @@ -2859,12 +2670,6 @@ public class TestDFSShell { if (localFile.exists()) localFile.delete(); - - if (null != fs) { - fs.delete(hdfsTestDir, true); - fs.close(); - } - cluster.shutdown(); } } @@ -2876,11 +2681,7 @@ public class TestDFSShell { */ @Test (timeout = 30000) public void testCopyFromLocalWithPermissionDenied() throws Exception { - Configuration conf = new Configuration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) - .format(true).build(); FsShell shell = null; - FileSystem fs = null; PrintStream bak = null; final File localFile = new File(TEST_ROOT_DIR, "testFileWithNoReadPermissions"); @@ -2889,11 +2690,10 @@ public class TestDFSShell { + counter.getAndIncrement(); final Path hdfsTestDir = new Path(testdir); try { - fs = cluster.getFileSystem(); - fs.mkdirs(hdfsTestDir); + dfs.mkdirs(hdfsTestDir); localFile.createNewFile(); localFile.setReadable(false); - writeFile(fs, new Path(testdir, "testFileForPut")); + writeFile(dfs, new Path(testdir, "testFileForPut")); shell = new FsShell(); // capture system error messages, snarfed from testErrOutPut() @@ -2929,17 +2729,13 @@ public class TestDFSShell { if (localFile.exists()) localFile.delete(); - if (null != fs) { - fs.delete(hdfsTestDir, true); - fs.close(); - } - cluster.shutdown(); + dfs.delete(hdfsTestDir, true); } } /** * Test -setrep with a replication factor that is too low. We have to test - * this here because the mini-cluster used with testHDFSConf.xml uses a + * this here because the mini-miniCluster used with testHDFSConf.xml uses a * replication factor of 1 (for good reason). */ @Test (timeout = 30000) @@ -2998,49 +2794,39 @@ public class TestDFSShell { // setrep for file and directory. @Test (timeout = 30000) public void testSetrep() throws Exception { - - Configuration conf = new Configuration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) - .format(true).build(); FsShell shell = null; - FileSystem fs = null; - final String testdir1 = "/tmp/TestDFSShell-testSetrep-" + counter.getAndIncrement(); final String testdir2 = testdir1 + "/nestedDir"; final Path hdfsFile1 = new Path(testdir1, "testFileForSetrep"); final Path hdfsFile2 = new Path(testdir2, "testFileForSetrep"); - final Short oldRepFactor = new Short((short) 1); + final Short oldRepFactor = new Short((short) 2); final Short newRepFactor = new Short((short) 3); try { String[] argv; - cluster.waitActive(); - fs = cluster.getFileSystem(); - assertThat(fs.mkdirs(new Path(testdir2)), is(true)); - shell = new FsShell(conf); + assertThat(dfs.mkdirs(new Path(testdir2)), is(true)); + shell = new FsShell(dfs.getConf()); - fs.create(hdfsFile1, true).close(); - fs.create(hdfsFile2, true).close(); + dfs.create(hdfsFile1, true).close(); + dfs.create(hdfsFile2, true).close(); // Tests for setrep on a file. argv = new String[] { "-setrep", newRepFactor.toString(), hdfsFile1.toString() }; assertThat(shell.run(argv), is(SUCCESS)); - assertThat(fs.getFileStatus(hdfsFile1).getReplication(), is(newRepFactor)); - assertThat(fs.getFileStatus(hdfsFile2).getReplication(), is(oldRepFactor)); + assertThat(dfs.getFileStatus(hdfsFile1).getReplication(), is(newRepFactor)); + assertThat(dfs.getFileStatus(hdfsFile2).getReplication(), is(oldRepFactor)); // Tests for setrep // Tests for setrep on a directory and make sure it is applied recursively. argv = new String[] { "-setrep", newRepFactor.toString(), testdir1 }; assertThat(shell.run(argv), is(SUCCESS)); - assertThat(fs.getFileStatus(hdfsFile1).getReplication(), is(newRepFactor)); - assertThat(fs.getFileStatus(hdfsFile2).getReplication(), is(newRepFactor)); + assertThat(dfs.getFileStatus(hdfsFile1).getReplication(), is(newRepFactor)); + assertThat(dfs.getFileStatus(hdfsFile2).getReplication(), is(newRepFactor)); } finally { if (shell != null) { shell.close(); } - - cluster.shutdown(); } } @@ -3049,7 +2835,7 @@ public class TestDFSShell { */ private void deleteFileUsingTrash( boolean serverTrash, boolean clientTrash) throws Exception { - // Run a cluster, optionally with trash enabled on the server + // Run a miniCluster, optionally with trash enabled on the server Configuration serverConf = new HdfsConfiguration(); if (serverTrash) { serverConf.setLong(FS_TRASH_INTERVAL_KEY, 1); @@ -3157,56 +2943,38 @@ public class TestDFSShell { File file1 = new File(testRoot, "file1"); createLocalFileWithRandomData(inputFileLength, file1); - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); + // Run appendToFile with insufficient arguments. + FsShell shell = new FsShell(); + shell.setConf(dfs.getConf()); + String[] argv = new String[] { + "-appendToFile", file1.toString() }; + int res = ToolRunner.run(shell, argv); + assertThat(res, not(0)); - try { - FileSystem dfs = cluster.getFileSystem(); - assertTrue("Not a HDFS: " + dfs.getUri(), - dfs instanceof DistributedFileSystem); - - // Run appendToFile with insufficient arguments. - FsShell shell = new FsShell(); - shell.setConf(conf); - String[] argv = new String[] { - "-appendToFile", file1.toString() }; - int res = ToolRunner.run(shell, argv); - assertThat(res, not(0)); - - // Mix stdin with other input files. Must fail. - Path remoteFile = new Path("/remoteFile"); - argv = new String[] { - "-appendToFile", file1.toString(), "-", remoteFile.toString() }; - res = ToolRunner.run(shell, argv); - assertThat(res, not(0)); - } finally { - cluster.shutdown(); - } + // Mix stdin with other input files. Must fail. + Path remoteFile = new Path("/remoteFile"); + argv = new String[] { + "-appendToFile", file1.toString(), "-", remoteFile.toString() }; + res = ToolRunner.run(shell, argv); + assertThat(res, not(0)); } @Test (timeout = 30000) public void testSetXAttrPermission() throws Exception { UserGroupInformation user = UserGroupInformation. createUserForTesting("user", new String[] {"mygroup"}); - MiniDFSCluster cluster = null; PrintStream bak = null; try { - final Configuration conf = new HdfsConfiguration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - - FileSystem fs = cluster.getFileSystem(); Path p = new Path("/foo"); - fs.mkdirs(p); + dfs.mkdirs(p); bak = System.err; - final FsShell fshell = new FsShell(conf); + final FsShell fshell = new FsShell(dfs.getConf()); final ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setErr(new PrintStream(out)); // No permission to write xattr - fs.setPermission(p, new FsPermission((short) 0700)); + dfs.setPermission(p, new FsPermission((short) 0700)); user.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { @@ -3227,7 +2995,7 @@ public class TestDFSShell { out.reset(); // No permission to read and remove - fs.setPermission(p, new FsPermission((short) 0750)); + dfs.setPermission(p, new FsPermission((short) 0750)); user.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { @@ -3254,28 +3022,19 @@ public class TestDFSShell { if (bak != null) { System.setErr(bak); } - if (cluster != null) { - cluster.shutdown(); - } } } /* HDFS-6413 xattr names erroneously handled as case-insensitive */ @Test (timeout = 30000) public void testSetXAttrCaseSensitivity() throws Exception { - MiniDFSCluster cluster = null; PrintStream bak = null; try { - final Configuration conf = new HdfsConfiguration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - - FileSystem fs = cluster.getFileSystem(); Path p = new Path("/mydir"); - fs.mkdirs(p); + dfs.mkdirs(p); bak = System.err; - final FsShell fshell = new FsShell(conf); + final FsShell fshell = new FsShell(dfs.getConf()); final ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setOut(new PrintStream(out)); @@ -3325,9 +3084,6 @@ public class TestDFSShell { if (bak != null) { System.setOut(bak); } - if (cluster != null) { - cluster.shutdown(); - } } } @@ -3387,6 +3143,7 @@ public class TestDFSShell { */ @Test (timeout = 30000) public void testSetXAttrPermissionAsDifferentOwner() throws Exception { + final String root = "/testSetXAttrPermissionAsDifferentOwner"; final String USER1 = "user1"; final String GROUP1 = "supergroup"; final UserGroupInformation user1 = UserGroupInformation. @@ -3394,18 +3151,13 @@ public class TestDFSShell { final UserGroupInformation user2 = UserGroupInformation. createUserForTesting("user2", new String[] {"mygroup2"}); final UserGroupInformation SUPERUSER = UserGroupInformation.getCurrentUser(); - MiniDFSCluster cluster = null; PrintStream bak = null; try { - final Configuration conf = new HdfsConfiguration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - - final FileSystem fs = cluster.getFileSystem(); - fs.setOwner(new Path("/"), USER1, GROUP1); + dfs.mkdirs(new Path(root)); + dfs.setOwner(new Path(root), USER1, GROUP1); bak = System.err; - final FsShell fshell = new FsShell(conf); + final FsShell fshell = new FsShell(dfs.getConf()); final ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setErr(new PrintStream(out)); @@ -3414,7 +3166,7 @@ public class TestDFSShell { @Override public Object run() throws Exception { final int ret = ToolRunner.run(fshell, new String[]{ - "-mkdir", "/foo"}); + "-mkdir", root + "/foo"}); assertEquals("Return should be 0", 0, ret); out.reset(); return null; @@ -3427,7 +3179,7 @@ public class TestDFSShell { public Object run() throws Exception { // Give access to "other" final int ret = ToolRunner.run(fshell, new String[]{ - "-chmod", "707", "/foo"}); + "-chmod", "707", root + "/foo"}); assertEquals("Return should be 0", 0, ret); out.reset(); return null; @@ -3440,7 +3192,7 @@ public class TestDFSShell { @Override public Object run() throws Exception { final int ret = ToolRunner.run(fshell, new String[]{ - "-setfattr", "-n", "user.a1", "-v", "1234", "/foo"}); + "-setfattr", "-n", "user.a1", "-v", "1234", root + "/foo"}); assertEquals("Returned should be 0", 0, ret); out.reset(); return null; @@ -3453,7 +3205,7 @@ public class TestDFSShell { @Override public Object run() throws Exception { final int ret = ToolRunner.run(fshell, new String[]{ - "-setfattr", "-n", "user.a1", "-v", "1234", "/foo"}); + "-setfattr", "-n", "user.a1", "-v", "1234", root + "/foo"}); assertEquals("Returned should be 0", 0, ret); out.reset(); return null; @@ -3467,12 +3219,12 @@ public class TestDFSShell { public Object run() throws Exception { // Read int ret = ToolRunner.run(fshell, new String[] { "-getfattr", "-n", - "user.a1", "/foo" }); + "user.a1", root + "/foo" }); assertEquals("Returned should be 0", 0, ret); out.reset(); // Remove ret = ToolRunner.run(fshell, new String[] { "-setfattr", "-x", - "user.a1", "/foo" }); + "user.a1", root + "/foo" }); assertEquals("Returned should be 0", 0, ret); out.reset(); return null; @@ -3494,7 +3246,7 @@ public class TestDFSShell { public Object run() throws Exception { // Give access to "other" final int ret = ToolRunner.run(fshell, new String[]{ - "-chmod", "700", "/foo"}); + "-chmod", "700", root + "/foo"}); assertEquals("Return should be 0", 0, ret); out.reset(); return null; @@ -3508,7 +3260,7 @@ public class TestDFSShell { public Object run() throws Exception { // set int ret = ToolRunner.run(fshell, new String[] { "-setfattr", "-n", - "user.a2", "/foo" }); + "user.a2", root + "/foo" }); assertEquals("Returned should be 1", 1, ret); final String str = out.toString(); assertTrue("Permission denied printed", @@ -3525,7 +3277,7 @@ public class TestDFSShell { public Object run() throws Exception { // set int ret = ToolRunner.run(fshell, new String[] { "-setfattr", "-x", - "user.a2", "/foo" }); + "user.a2", root + "/foo" }); assertEquals("Returned should be 1", 1, ret); final String str = out.toString(); assertTrue("Permission denied printed", @@ -3541,7 +3293,7 @@ public class TestDFSShell { public Object run() throws Exception { // set int ret = ToolRunner.run(fshell, new String[] { "-setfattr", "-n", - "trusted.a3", "/foo" }); + "trusted.a3", root + "/foo" }); assertEquals("Returned should be 0", 0, ret); out.reset(); return null; @@ -3551,9 +3303,6 @@ public class TestDFSShell { if (bak != null) { System.setErr(bak); } - if (cluster != null) { - cluster.shutdown(); - } } } @@ -3567,28 +3316,22 @@ public class TestDFSShell { public void testGetFAttrErrors() throws Exception { final UserGroupInformation user = UserGroupInformation. createUserForTesting("user", new String[] {"mygroup"}); - MiniDFSCluster cluster = null; PrintStream bakErr = null; try { - final Configuration conf = new HdfsConfiguration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - - final FileSystem fs = cluster.getFileSystem(); - final Path p = new Path("/foo"); - fs.mkdirs(p); + final Path p = new Path("/testGetFAttrErrors"); + dfs.mkdirs(p); bakErr = System.err; - final FsShell fshell = new FsShell(conf); + final FsShell fshell = new FsShell(dfs.getConf()); final ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setErr(new PrintStream(out)); // No permission for "other". - fs.setPermission(p, new FsPermission((short) 0700)); + dfs.setPermission(p, new FsPermission((short) 0700)); { final int ret = ToolRunner.run(fshell, new String[] { - "-setfattr", "-n", "user.a1", "-v", "1234", "/foo"}); + "-setfattr", "-n", "user.a1", "-v", "1234", p.toString()}); assertEquals("Returned should be 0", 0, ret); out.reset(); } @@ -3597,7 +3340,7 @@ public class TestDFSShell { @Override public Object run() throws Exception { int ret = ToolRunner.run(fshell, new String[] { - "-getfattr", "-n", "user.a1", "/foo"}); + "-getfattr", "-n", "user.a1", p.toString()}); String str = out.toString(); assertTrue("xattr value was incorrectly returned", str.indexOf("1234") == -1); @@ -3608,7 +3351,7 @@ public class TestDFSShell { { final int ret = ToolRunner.run(fshell, new String[]{ - "-getfattr", "-n", "user.nonexistent", "/foo"}); + "-getfattr", "-n", "user.nonexistent", p.toString()}); String str = out.toString(); assertTrue("xattr value was incorrectly returned", str.indexOf( @@ -3620,9 +3363,6 @@ public class TestDFSShell { if (bakErr != null) { System.setErr(bakErr); } - if (cluster != null) { - cluster.shutdown(); - } } } @@ -3705,48 +3445,34 @@ public class TestDFSShell { @Test (timeout = 30000) public void testMkdirReserved() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = cluster.getFileSystem(); try { - fs.mkdirs(new Path("/.reserved")); + dfs.mkdirs(new Path("/.reserved")); fail("Can't mkdir /.reserved"); } catch (Exception e) { // Expected, HadoopIllegalArgumentException thrown from remote assertTrue(e.getMessage().contains("\".reserved\" is reserved")); } - cluster.shutdown(); } @Test (timeout = 30000) public void testRmReserved() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = cluster.getFileSystem(); try { - fs.delete(new Path("/.reserved"), true); + dfs.delete(new Path("/.reserved"), true); fail("Can't delete /.reserved"); } catch (Exception e) { // Expected, InvalidPathException thrown from remote assertTrue(e.getMessage().contains("Invalid path name /.reserved")); } - cluster.shutdown(); } @Test //(timeout = 30000) public void testCopyReserved() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = cluster.getFileSystem(); final File localFile = new File(TEST_ROOT_DIR, "testFileForPut"); localFile.createNewFile(); final String localfilepath = new Path(localFile.getAbsolutePath()).toUri().toString(); try { - fs.copyFromLocalFile(new Path(localfilepath), new Path("/.reserved")); + dfs.copyFromLocalFile(new Path(localfilepath), new Path("/.reserved")); fail("Can't copyFromLocal to /.reserved"); } catch (Exception e) { // Expected, InvalidPathException thrown from remote @@ -3756,10 +3482,10 @@ public class TestDFSShell { final String testdir = GenericTestUtils.getTempPath( "TestDFSShell-testCopyReserved"); final Path hdfsTestDir = new Path(testdir); - writeFile(fs, new Path(testdir, "testFileForPut")); + writeFile(dfs, new Path(testdir, "testFileForPut")); final Path src = new Path(hdfsTestDir, "srcfile"); - fs.create(src).close(); - assertTrue(fs.exists(src)); + dfs.create(src).close(); + assertTrue(dfs.exists(src)); // runCmd prints error into System.err, thus verify from there. PrintStream syserr = System.err; @@ -3767,97 +3493,71 @@ public class TestDFSShell { PrintStream ps = new PrintStream(baos); System.setErr(ps); try { - FsShell shell = new FsShell(); - shell.setConf(conf); + FsShell shell = new FsShell(dfs.getConf()); runCmd(shell, "-cp", src.toString(), "/.reserved"); assertTrue(baos.toString().contains("Invalid path name /.reserved")); } finally { System.setErr(syserr); - cluster.shutdown(); } } @Test (timeout = 30000) public void testChmodReserved() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = cluster.getFileSystem(); - // runCmd prints error into System.err, thus verify from there. PrintStream syserr = System.err; final ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos); System.setErr(ps); try { - FsShell shell = new FsShell(); - shell.setConf(conf); + FsShell shell = new FsShell(dfs.getConf()); runCmd(shell, "-chmod", "777", "/.reserved"); assertTrue(baos.toString().contains("Invalid path name /.reserved")); } finally { System.setErr(syserr); - cluster.shutdown(); } } @Test (timeout = 30000) public void testChownReserved() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = cluster.getFileSystem(); - // runCmd prints error into System.err, thus verify from there. PrintStream syserr = System.err; final ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos); System.setErr(ps); try { - FsShell shell = new FsShell(); - shell.setConf(conf); + FsShell shell = new FsShell(dfs.getConf()); runCmd(shell, "-chown", "user1", "/.reserved"); assertTrue(baos.toString().contains("Invalid path name /.reserved")); } finally { System.setErr(syserr); - cluster.shutdown(); } } @Test (timeout = 30000) public void testSymLinkReserved() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - FileSystem fs = cluster.getFileSystem(); try { - fs.createSymlink(new Path("/.reserved"), new Path("/rl1"), false); + dfs.createSymlink(new Path("/.reserved"), new Path("/rl1"), false); fail("Can't create symlink to /.reserved"); } catch (Exception e) { // Expected, InvalidPathException thrown from remote assertTrue(e.getMessage().contains("Invalid target name: /.reserved")); } - cluster.shutdown(); } @Test (timeout = 30000) public void testSnapshotReserved() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - DistributedFileSystem fs = cluster.getFileSystem(); final Path reserved = new Path("/.reserved"); try { - fs.allowSnapshot(reserved); + dfs.allowSnapshot(reserved); fail("Can't allow snapshot on /.reserved"); } catch (FileNotFoundException e) { assertTrue(e.getMessage().contains("Directory does not exist")); } try { - fs.createSnapshot(reserved, "snap"); + dfs.createSnapshot(reserved, "snap"); fail("Can't create snapshot on /.reserved"); } catch (FileNotFoundException e) { assertTrue(e.getMessage().contains("Directory/File does not exist")); } - cluster.shutdown(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 9168ca6c890..67019c31c24 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs; import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; import java.io.RandomAccessFile; @@ -97,6 +96,7 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; +import static org.junit.Assert.assertNotNull; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.anyObject; @@ -604,13 +604,8 @@ public class TestEncryptionZones { assertExceptionContains("Permission denied:", e); } - try { - userAdmin.getEncryptionZoneForPath(nonexistent); - fail("FileNotFoundException should be thrown for a non-existent" - + " file path"); - } catch (FileNotFoundException e) { - assertExceptionContains("Path not found: " + nonexistent, e); - } + assertNull("expected null for nonexistent path", + userAdmin.getEncryptionZoneForPath(nonexistent)); // Check operation with non-ez paths assertNull("expected null for non-ez path", @@ -638,20 +633,10 @@ public class TestEncryptionZones { assertEquals("expected ez path", allPath.toString(), userAdmin.getEncryptionZoneForPath( new Path(snapshottedAllPath)).getPath().toString()); - try { - userAdmin.getEncryptionZoneForPath(allPathFile); - fail("FileNotFoundException should be thrown for a non-existent" - + " file path"); - } catch (FileNotFoundException e) { - assertExceptionContains("Path not found: " + allPathFile, e); - } - try { - userAdmin.getEncryptionZoneForPath(allPath); - fail("FileNotFoundException should be thrown for a non-existent" - + " file path"); - } catch (FileNotFoundException e) { - assertExceptionContains("Path not found: " + allPath, e); - } + assertNull("expected null for deleted file path", + userAdmin.getEncryptionZoneForPath(allPathFile)); + assertNull("expected null for deleted directory path", + userAdmin.getEncryptionZoneForPath(allPath)); return null; } }); @@ -1062,7 +1047,7 @@ public class TestEncryptionZones { } private class MyInjector extends EncryptionFaultInjector { - int generateCount; + volatile int generateCount; CountDownLatch ready; CountDownLatch wait; @@ -1072,13 +1057,27 @@ public class TestEncryptionZones { } @Override - public void startFileAfterGenerateKey() throws IOException { + public void startFileNoKey() throws IOException { + generateCount = -1; + syncWithLatches(); + } + + @Override + public void startFileBeforeGenerateKey() throws IOException { + syncWithLatches(); + } + + private void syncWithLatches() throws IOException { ready.countDown(); try { wait.await(); } catch (InterruptedException e) { throw new IOException(e); } + } + + @Override + public void startFileAfterGenerateKey() throws IOException { generateCount++; } } @@ -1114,10 +1113,14 @@ public class TestEncryptionZones { Future future = executor.submit(new CreateFileTask(fsWrapper, file)); injector.ready.await(); - // Do the fault - doFault(); - // Allow create to proceed - injector.wait.countDown(); + try { + // Do the fault + doFault(); + // Allow create to proceed + } finally { + // Always decrement latch to avoid hanging the tests on failure. + injector.wait.countDown(); + } future.get(); // Cleanup and postconditions doCleanup(); @@ -1140,20 +1143,21 @@ public class TestEncryptionZones { fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true); ExecutorService executor = Executors.newSingleThreadExecutor(); - // Test when the parent directory becomes an EZ + // Test when the parent directory becomes an EZ. With no initial EZ, + // the fsn lock must not be yielded. executor.submit(new InjectFaultTask() { - @Override - public void doFault() throws Exception { - dfsAdmin.createEncryptionZone(zone1, TEST_KEY, NO_TRASH); - } @Override public void doCleanup() throws Exception { - assertEquals("Expected a startFile retry", 2, injector.generateCount); + assertEquals("Expected no startFile key generation", + -1, injector.generateCount); fsWrapper.delete(file, false); } }).get(); - // Test when the parent directory unbecomes an EZ + // Test when the parent directory unbecomes an EZ. The generation of + // the EDEK will yield the lock, then re-resolve the path and use the + // previous EDEK. + dfsAdmin.createEncryptionZone(zone1, TEST_KEY, NO_TRASH); executor.submit(new InjectFaultTask() { @Override public void doFault() throws Exception { @@ -1166,7 +1170,9 @@ public class TestEncryptionZones { } }).get(); - // Test when the parent directory becomes a different EZ + // Test when the parent directory becomes a different EZ. The generation + // of the EDEK will yield the lock, re-resolve will detect the EZ has + // changed, and client will be asked to retry a 2nd time fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true); final String otherKey = "other_key"; DFSTestUtil.createKey(otherKey, cluster, conf); @@ -1498,25 +1504,18 @@ public class TestEncryptionZones { } @Test(timeout = 60000) - public void testGetEncryptionZoneOnANonExistentZoneFile() throws Exception { - final Path ez = new Path("/ez"); - fs.mkdirs(ez); - dfsAdmin.createEncryptionZone(ez, TEST_KEY, NO_TRASH); - Path zoneFile = new Path(ez, "file"); - try { - fs.getEZForPath(zoneFile); - fail("FileNotFoundException should be thrown for a non-existent" - + " file path"); - } catch (FileNotFoundException e) { - assertExceptionContains("Path not found: " + zoneFile, e); - } - try { - dfsAdmin.getEncryptionZoneForPath(zoneFile); - fail("FileNotFoundException should be thrown for a non-existent" - + " file path"); - } catch (FileNotFoundException e) { - assertExceptionContains("Path not found: " + zoneFile, e); - } + public void testGetEncryptionZoneOnANonExistentPaths() throws Exception { + final Path ezPath = new Path("/ez"); + fs.mkdirs(ezPath); + dfsAdmin.createEncryptionZone(ezPath, TEST_KEY, NO_TRASH); + Path zoneFile = new Path(ezPath, "file"); + EncryptionZone ez = fs.getEZForPath(zoneFile); + assertNotNull("Expected EZ for non-existent path in EZ", ez); + ez = dfsAdmin.getEncryptionZoneForPath(zoneFile); + assertNotNull("Expected EZ for non-existent path in EZ", ez); + ez = dfsAdmin.getEncryptionZoneForPath( + new Path("/does/not/exist")); + assertNull("Expected null for non-existent path not in EZ", ez); } @Test(timeout = 120000) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java index 41478516010..ae0f0c274d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -89,7 +90,7 @@ public class TestFileAppend4 { // handle failures in the DFSClient pipeline quickly // (for cluster.shutdown(); fs.close() idiom) - conf.setInt("ipc.client.connect.max.retries", 1); + conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1); } /* diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java index 2437e38acc9..5477700fa07 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs; +import com.google.common.base.Supplier; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import static org.junit.Assert.assertEquals; @@ -38,6 +39,7 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ChecksumException; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportReplica; @@ -230,6 +232,65 @@ public class TestFileCorruption { } + @Test + public void testSetReplicationWhenBatchIBR() throws Exception { + Configuration conf = new HdfsConfiguration(); + conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 100); + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INCREMENTAL_INTERVAL_MSEC_KEY, + 30000); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_FILE_CLOSE_NUM_COMMITTED_ALLOWED_KEY, + 1); + DistributedFileSystem dfs; + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(3).build()) { + final int bufferSize = 1024; // 1024 Bytes each time + byte[] outBuffer = new byte[bufferSize]; + dfs = cluster.getFileSystem(); + String fileName = "/testSetRep1"; + Path filePath = new Path(fileName); + FSDataOutputStream out = dfs.create(filePath); + out.write(outBuffer, 0, bufferSize); + out.close(); + //sending the FBR to Delay next IBR + cluster.triggerBlockReports(); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + try { + cluster.triggerBlockReports(); + if (cluster.getNamesystem().getBlocksTotal() == 1) { + return true; + } + } catch (Exception e) { + // Ignore the exception + } + return false; + } + }, 10, 3000); + fileName = "/testSetRep2"; + filePath = new Path(fileName); + out = dfs.create(filePath); + out.write(outBuffer, 0, bufferSize); + out.close(); + dfs.setReplication(filePath, (short) 10); + // underreplicated Blocks should be one after setrep + GenericTestUtils.waitFor(new Supplier() { + @Override public Boolean get() { + try { + return cluster.getNamesystem().getBlockManager() + .getUnderReplicatedBlocksCount() == 1; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + }, 10, 3000); + assertEquals(0, + cluster.getNamesystem().getBlockManager().getMissingBlocksCount()); + } + } + private void markAllBlocksAsCorrupt(BlockManager bm, ExtendedBlock blk) throws IOException { for (DatanodeStorageInfo info : bm.getStorages(blk.getLocalBlock())) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java index 87c3b4cb308..f7bac28cebf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java @@ -166,6 +166,7 @@ public class TestLeaseRecoveryStriped { // After recovery, storages are reported by primary DN. we should verify // storages reported by blockReport. cluster.restartNameNode(true); + cluster.waitFirstBRCompleted(0, 10000); StripedFileTestUtil.checkData(dfs, p, safeLength, new ArrayList(), oldGS); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java index 9374ae8efe1..5a8a39a89e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -218,7 +219,7 @@ public class TestBlockTokenWithDFS { conf.setInt("io.bytes.per.checksum", BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, numDataNodes); - conf.setInt("ipc.client.connect.max.retries", 0); + conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); // Set short retry timeouts so this test runs faster conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10); return conf; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java index b11b48aed79..6efc53a8746 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; +import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi.FsVolumeReferences; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -216,13 +217,13 @@ public class TestNameNodePrunesMissingStorages { datanodeToRemoveStorageFromIdx++; } // Find the volume within the datanode which holds that first storage. - String volumeDirectoryToRemove = null; + StorageLocation volumeLocationToRemove = null; try (FsVolumeReferences volumes = datanodeToRemoveStorageFrom.getFSDataset().getFsVolumeReferences()) { assertEquals(NUM_STORAGES_PER_DN, volumes.size()); for (FsVolumeSpi volume : volumes) { if (volume.getStorageID().equals(storageIdToRemove)) { - volumeDirectoryToRemove = volume.getBasePath(); + volumeLocationToRemove = volume.getStorageLocation(); } } }; @@ -230,10 +231,11 @@ public class TestNameNodePrunesMissingStorages { // Replace the volume directory with a regular file, which will // cause a volume failure. (If we merely removed the directory, // it would be re-initialized with a new storage ID.) - assertNotNull(volumeDirectoryToRemove); + assertNotNull(volumeLocationToRemove); datanodeToRemoveStorageFrom.shutdown(); - FileUtil.fullyDelete(new File(volumeDirectoryToRemove)); - FileOutputStream fos = new FileOutputStream(volumeDirectoryToRemove); + FileUtil.fullyDelete(volumeLocationToRemove.getFile()); + FileOutputStream fos = new FileOutputStream( + volumeLocationToRemove.getFile().toString()); try { fos.write(1); } finally { @@ -326,7 +328,8 @@ public class TestNameNodePrunesMissingStorages { dn.getFSDataset().getFsVolumeReferences(); final String newStorageId = DatanodeStorage.generateUuid(); try { - File currentDir = new File(volumeRefs.get(0).getBasePath(), "current"); + File currentDir = new File( + volumeRefs.get(0).getStorageLocation().getFile(), "current"); File versionFile = new File(currentDir, "VERSION"); rewriteVersionFile(versionFile, newStorageId); } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java index 152e153174f..3bc13a8c36d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java @@ -35,12 +35,14 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; import org.junit.After; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -58,57 +60,44 @@ public class TestReconstructStripedBlocksWithRackAwareness { GenericTestUtils.setLogLevel(BlockManager.LOG, Level.ALL); } - private static final String[] hosts = getHosts(); - private static final String[] racks = getRacks(); + private static final String[] hosts = + getHosts(NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 1); + private static final String[] racks = + getRacks(NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 1, NUM_DATA_BLOCKS); - private static String[] getHosts() { - String[] hosts = new String[NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 1]; + private static String[] getHosts(int numHosts) { + String[] hosts = new String[numHosts]; for (int i = 0; i < hosts.length; i++) { hosts[i] = "host" + (i + 1); } return hosts; } - private static String[] getRacks() { - String[] racks = new String[NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 1]; - int numHostEachRack = (NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS - 1) / - (NUM_DATA_BLOCKS - 1) + 1; + private static String[] getRacks(int numHosts, int numRacks) { + String[] racks = new String[numHosts]; + int numHostEachRack = numHosts / numRacks; + int residue = numHosts % numRacks; int j = 0; - // we have NUM_DATA_BLOCKS racks - for (int i = 1; i <= NUM_DATA_BLOCKS; i++) { - if (j == racks.length - 1) { - assert i == NUM_DATA_BLOCKS; + for (int i = 1; i <= numRacks; i++) { + int limit = i <= residue ? numHostEachRack + 1 : numHostEachRack; + for (int k = 0; k < limit; k++) { racks[j++] = "/r" + i; - } else { - for (int k = 0; k < numHostEachRack && j < racks.length - 1; k++) { - racks[j++] = "/r" + i; - } } } + assert j == numHosts; return racks; } private MiniDFSCluster cluster; + private static final HdfsConfiguration conf = new HdfsConfiguration(); private DistributedFileSystem fs; - private FSNamesystem fsn; - private BlockManager bm; - @Before - public void setup() throws Exception { - final HdfsConfiguration conf = new HdfsConfiguration(); + @BeforeClass + public static void setup() throws Exception { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false); - - cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts) - .numDataNodes(hosts.length).build(); - cluster.waitActive(); - - fsn = cluster.getNamesystem(); - bm = fsn.getBlockManager(); - - fs = cluster.getFileSystem(); - fs.setErasureCodingPolicy(new Path("/"), null); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 1); } @After @@ -132,6 +121,15 @@ public class TestReconstructStripedBlocksWithRackAwareness { return dnProp; } + private DataNode getDataNode(String host) { + for (DataNode dn : cluster.getDataNodes()) { + if (dn.getDatanodeId().getHostName().equals(host)) { + return dn; + } + } + return null; + } + /** * When there are all the internal blocks available but they are not placed on * enough racks, NameNode should avoid normal decoding reconstruction but copy @@ -143,9 +141,19 @@ public class TestReconstructStripedBlocksWithRackAwareness { */ @Test public void testReconstructForNotEnoughRacks() throws Exception { + LOG.info("cluster hosts: {}, racks: {}", Arrays.asList(hosts), + Arrays.asList(racks)); + + cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts) + .numDataNodes(hosts.length).build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + fs.setErasureCodingPolicy(new Path("/"), null); + FSNamesystem fsn = cluster.getNamesystem(); + BlockManager bm = fsn.getBlockManager(); + MiniDFSCluster.DataNodeProperties lastHost = stopDataNode( hosts[hosts.length - 1]); - final Path file = new Path("/foo"); // the file's block is in 9 dn but 5 racks DFSTestUtil.createFile(fs, file, @@ -206,6 +214,12 @@ public class TestReconstructStripedBlocksWithRackAwareness { @Test public void testChooseExcessReplicasToDelete() throws Exception { + cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts) + .numDataNodes(hosts.length).build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + fs.setErasureCodingPolicy(new Path("/"), null); + MiniDFSCluster.DataNodeProperties lastHost = stopDataNode( hosts[hosts.length - 1]); @@ -242,4 +256,82 @@ public class TestReconstructStripedBlocksWithRackAwareness { Assert.assertFalse(dn.getHostName().equals("host1")); } } + + /** + * In case we have 10 internal blocks on 5 racks, where 9 of blocks are live + * and 1 decommissioning, make sure the reconstruction happens correctly. + */ + @Test + public void testReconstructionWithDecommission() throws Exception { + final String[] racks = getRacks(NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 2, + NUM_DATA_BLOCKS); + final String[] hosts = getHosts(NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 2); + // we now have 11 hosts on 6 racks with distribution: 2-2-2-2-2-1 + cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts) + .numDataNodes(hosts.length).build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + fs.setErasureCodingPolicy(new Path("/"), null); + + final BlockManager bm = cluster.getNamesystem().getBlockManager(); + final DatanodeManager dm = bm.getDatanodeManager(); + + // stop h9 and h10 and create a file with 6+3 internal blocks + MiniDFSCluster.DataNodeProperties h9 = stopDataNode(hosts[hosts.length - 3]); + MiniDFSCluster.DataNodeProperties h10 = stopDataNode(hosts[hosts.length - 2]); + final Path file = new Path("/foo"); + DFSTestUtil.createFile(fs, file, + BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS * 2, (short) 1, 0L); + final BlockInfo blockInfo = cluster.getNamesystem().getFSDirectory() + .getINode(file.toString()).asFile().getLastBlock(); + + // bring h9 back + cluster.restartDataNode(h9); + cluster.waitActive(); + + // stop h11 so that the reconstruction happens + MiniDFSCluster.DataNodeProperties h11 = stopDataNode(hosts[hosts.length - 1]); + boolean recovered = bm.countNodes(blockInfo).liveReplicas() >= + NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS; + for (int i = 0; i < 10 & !recovered; i++) { + Thread.sleep(1000); + recovered = bm.countNodes(blockInfo).liveReplicas() >= + NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS; + } + Assert.assertTrue(recovered); + + // mark h9 as decommissioning + DataNode datanode9 = getDataNode(hosts[hosts.length - 3]); + Assert.assertNotNull(datanode9); + final DatanodeDescriptor dn9 = dm.getDatanode(datanode9.getDatanodeId()); + dn9.startDecommission(); + + // restart h10 and h11 + cluster.restartDataNode(h10); + cluster.restartDataNode(h11); + cluster.waitActive(); + DataNodeTestUtils.triggerBlockReport(getDataNode(hosts[hosts.length - 1])); + + // start decommissioning h9 + boolean satisfied = bm.isPlacementPolicySatisfied(blockInfo); + Assert.assertFalse(satisfied); + final DecommissionManager decomManager = + (DecommissionManager) Whitebox.getInternalState(dm, "decomManager"); + cluster.getNamesystem().writeLock(); + try { + dn9.stopDecommission(); + decomManager.startDecommission(dn9); + } finally { + cluster.getNamesystem().writeUnlock(); + } + + // make sure the decommission finishes and the block in on 6 racks + boolean decommissioned = dn9.isDecommissioned(); + for (int i = 0; i < 10 && !decommissioned; i++) { + Thread.sleep(1000); + decommissioned = dn9.isDecommissioned(); + } + Assert.assertTrue(decommissioned); + Assert.assertTrue(bm.isPlacementPolicySatisfied(blockInfo)); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 6034d1ee326..6c592310d65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -22,7 +22,9 @@ import java.io.FileDescriptor; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.URI; import java.nio.channels.ClosedChannelException; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -38,6 +40,7 @@ import javax.management.StandardMBean; import org.apache.commons.lang.ArrayUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.DF; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.util.AutoCloseableLock; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -46,6 +49,7 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; +import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ReportCompiler; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -494,21 +498,6 @@ public class SimulatedFSDataset implements FsDatasetSpi { return storage.getCapacity() - storage.getUsed(); } - @Override - public String getBasePath() { - return null; - } - - @Override - public String getPath(String bpid) throws IOException { - return null; - } - - @Override - public File getFinalizedDir(String bpid) throws IOException { - return null; - } - @Override public StorageType getStorageType() { return null; @@ -546,6 +535,28 @@ public class SimulatedFSDataset implements FsDatasetSpi { public FsDatasetSpi getDataset() { throw new UnsupportedOperationException(); } + + @Override + public StorageLocation getStorageLocation() { + return null; + } + + @Override + public URI getBaseURI() { + return null; + } + + @Override + public DF getUsageStats(Configuration conf) { + return null; + } + + @Override + public LinkedList compileReport(String bpid, + LinkedList report, ReportCompiler reportCompiler) + throws InterruptedException, IOException { + return null; + } } private final Map> blockMap @@ -1030,7 +1041,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { } @Override - public Set checkDataDir() { + public Set checkDataDir() { // nothing to check for simulated data set return null; } @@ -1344,7 +1355,8 @@ public class SimulatedFSDataset implements FsDatasetSpi { } @Override - public synchronized void removeVolumes(Set volumes, boolean clearFailure) { + public synchronized void removeVolumes(Collection volumes, + boolean clearFailure) { throw new UnsupportedOperationException(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java index 021361b2d8a..c55a82883ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java @@ -549,7 +549,8 @@ public class TestBlockScanner { info.shouldRun = false; } ctx.datanode.shutdown(); - String vPath = ctx.volumes.get(0).getBasePath(); + String vPath = ctx.volumes.get(0).getStorageLocation() + .getFile().getAbsolutePath(); File cursorPath = new File(new File(new File(vPath, "current"), ctx.bpids[0]), "scanner.cursor"); assertTrue("Failed to find cursor save file in " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java index 0dbb09c4d16..06387c5968e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java @@ -52,7 +52,6 @@ import org.junit.Test; import java.io.File; import java.io.IOException; -import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -519,11 +518,8 @@ public class TestDataNodeHotSwapVolumes { ExtendedBlock block = DFSTestUtil.getAllBlocks(fs, testFile).get(1).getBlock(); FsVolumeSpi volumeWithBlock = dn.getFSDataset().getVolume(block); - String basePath = volumeWithBlock.getBasePath(); - File storageDir = new File(basePath); - URI fileUri = storageDir.toURI(); - String dirWithBlock = - "[" + volumeWithBlock.getStorageType() + "]" + fileUri; + String dirWithBlock = "[" + volumeWithBlock.getStorageType() + "]" + + volumeWithBlock.getStorageLocation().getFile().toURI(); String newDirs = dirWithBlock; for (String dir : oldDirs) { if (dirWithBlock.startsWith(dir)) { @@ -581,8 +577,8 @@ public class TestDataNodeHotSwapVolumes { try (FsDatasetSpi.FsVolumeReferences volumes = dataset.getFsVolumeReferences()) { for (FsVolumeSpi volume : volumes) { - assertThat(volume.getBasePath(), is(not(anyOf( - is(newDirs.get(0)), is(newDirs.get(2)))))); + assertThat(volume.getStorageLocation().getFile().toString(), + is(not(anyOf(is(newDirs.get(0)), is(newDirs.get(2)))))); } } DataStorage storage = dn.getStorage(); @@ -765,7 +761,7 @@ public class TestDataNodeHotSwapVolumes { try (FsDatasetSpi.FsVolumeReferences volumes = dn.getFSDataset().getFsVolumeReferences()) { for (FsVolumeSpi vol : volumes) { - if (vol.getBasePath().equals(basePath.getPath())) { + if (vol.getBaseURI().equals(basePath.toURI())) { return (FsVolumeImpl) vol; } } @@ -810,6 +806,7 @@ public class TestDataNodeHotSwapVolumes { assertEquals(used, failedVolume.getDfsUsed()); DataNodeTestUtils.restoreDataDirFromFailure(dirToFail); + LOG.info("reconfiguring DN "); assertThat( "DN did not update its own config", dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, oldDataDir), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java index 6792ba8af78..47f48231b4b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java @@ -21,7 +21,6 @@ import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -254,17 +253,18 @@ public class TestDataNodeVolumeFailure { FsDatasetSpi data = dn0.getFSDataset(); try (FsDatasetSpi.FsVolumeReferences vols = data.getFsVolumeReferences()) { for (FsVolumeSpi volume : vols) { - assertNotEquals(new File(volume.getBasePath()).getAbsoluteFile(), - dn0Vol1.getAbsoluteFile()); + assertFalse(volume.getStorageLocation().getFile() + .getAbsolutePath().startsWith(dn0Vol1.getAbsolutePath() + )); } } // 3. all blocks on dn0Vol1 have been removed. for (ReplicaInfo replica : FsDatasetTestUtil.getReplicas(data, bpid)) { assertNotNull(replica.getVolume()); - assertNotEquals( - new File(replica.getVolume().getBasePath()).getAbsoluteFile(), - dn0Vol1.getAbsoluteFile()); + assertFalse(replica.getVolume().getStorageLocation().getFile() + .getAbsolutePath().startsWith(dn0Vol1.getAbsolutePath() + )); } // 4. dn0Vol1 is not in DN0's configuration and dataDirs anymore. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java index 8d021cd0026..4bb5e7abad2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java @@ -539,6 +539,16 @@ public class TestDataNodeVolumeFailureReporting { assertCounter("VolumeFailures", expectedVolumeFailuresCounter, getMetrics(dn.getMetrics().name())); FsDatasetSpi fsd = dn.getFSDataset(); + StringBuilder strBuilder = new StringBuilder(); + strBuilder.append("expectedFailedVolumes is "); + for (String expected: expectedFailedVolumes) { + strBuilder.append(expected + ","); + } + strBuilder.append(" fsd.getFailedStorageLocations() is "); + for (String expected: fsd.getFailedStorageLocations()) { + strBuilder.append(expected + ","); + } + LOG.info(strBuilder.toString()); assertEquals(expectedFailedVolumes.length, fsd.getNumFailedVolumes()); assertArrayEquals(expectedFailedVolumes, fsd.getFailedStorageLocations()); if (expectedFailedVolumes.length > 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index 576aae066a1..08a5af9d62b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -28,6 +28,7 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.net.URI; import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; import java.util.ArrayList; @@ -44,6 +45,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.DF; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.StorageType; @@ -56,11 +58,13 @@ import org.apache.hadoop.util.AutoCloseableLock; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ReportCompiler; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi.FsVolumeReferences; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.LazyPersistTestCase; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; @@ -185,18 +189,20 @@ public class TestDirectoryScanner { // Volume without a copy of the block. Make a copy now. File sourceBlock = new File(b.getBlockURI()); File sourceMeta = new File(b.getMetadataURI()); - String sourceRoot = b.getVolume().getBasePath(); - String destRoot = v.getBasePath(); + URI sourceRoot = b.getVolume().getStorageLocation().getFile().toURI(); + URI destRoot = v.getStorageLocation().getFile().toURI(); String relativeBlockPath = - new File(sourceRoot).toURI().relativize(sourceBlock.toURI()) + sourceRoot.relativize(sourceBlock.toURI()) .getPath(); String relativeMetaPath = - new File(sourceRoot).toURI().relativize(sourceMeta.toURI()) + sourceRoot.relativize(sourceMeta.toURI()) .getPath(); - File destBlock = new File(destRoot, relativeBlockPath); - File destMeta = new File(destRoot, relativeMetaPath); + File destBlock = new File(new File(destRoot).toString(), + relativeBlockPath); + File destMeta = new File(new File(destRoot).toString(), + relativeMetaPath); destBlock.getParentFile().mkdirs(); FileUtils.copyFile(sourceBlock, destBlock); @@ -238,7 +244,8 @@ public class TestDirectoryScanner { try (FsDatasetSpi.FsVolumeReferences volumes = fds.getFsVolumeReferences()) { int numVolumes = volumes.size(); int index = rand.nextInt(numVolumes - 1); - File finalizedDir = volumes.get(index).getFinalizedDir(bpid); + File finalizedDir = ((FsVolumeImpl) volumes.get(index)) + .getFinalizedDir(bpid); File file = new File(finalizedDir, getBlockFile(id)); if (file.createNewFile()) { LOG.info("Created block file " + file.getName()); @@ -253,8 +260,8 @@ public class TestDirectoryScanner { try (FsDatasetSpi.FsVolumeReferences refs = fds.getFsVolumeReferences()) { int numVolumes = refs.size(); int index = rand.nextInt(numVolumes - 1); - - File finalizedDir = refs.get(index).getFinalizedDir(bpid); + File finalizedDir = ((FsVolumeImpl) refs.get(index)) + .getFinalizedDir(bpid); File file = new File(finalizedDir, getMetaFile(id)); if (file.createNewFile()) { LOG.info("Created metafile " + file.getName()); @@ -271,7 +278,8 @@ public class TestDirectoryScanner { int numVolumes = refs.size(); int index = rand.nextInt(numVolumes - 1); - File finalizedDir = refs.get(index).getFinalizedDir(bpid); + File finalizedDir = + ((FsVolumeImpl) refs.get(index)).getFinalizedDir(bpid); File file = new File(finalizedDir, getBlockFile(id)); if (file.createNewFile()) { LOG.info("Created block file " + file.getName()); @@ -311,7 +319,7 @@ public class TestDirectoryScanner { scanner.reconcile(); assertTrue(scanner.diffs.containsKey(bpid)); - LinkedList diff = scanner.diffs.get(bpid); + LinkedList diff = scanner.diffs.get(bpid); assertTrue(scanner.stats.containsKey(bpid)); DirectoryScanner.Stats stats = scanner.stats.get(bpid); @@ -820,17 +828,6 @@ public class TestDirectoryScanner { return 0; } - @Override - public String getBasePath() { - return (new File("/base")).getAbsolutePath(); - } - - @Override - public String getPath(String bpid) throws IOException { - return (new File("/base/current/" + bpid)).getAbsolutePath(); - } - - @Override public File getFinalizedDir(String bpid) throws IOException { return new File("/base/current/" + bpid + "/finalized"); } @@ -877,6 +874,29 @@ public class TestDirectoryScanner { public FsDatasetSpi getDataset() { throw new UnsupportedOperationException(); } + + @Override + public StorageLocation getStorageLocation() { + return null; + } + + @Override + public URI getBaseURI() { + return (new File("/base")).toURI(); + } + + @Override + public DF getUsageStats(Configuration conf) { + return null; + } + + @Override + public LinkedList compileReport(String bpid, + LinkedList report, ReportCompiler reportCompiler) + throws InterruptedException, IOException { + return null; + } + } private final static TestFsVolumeSpi TEST_VOLUME = new TestFsVolumeSpi(); @@ -887,8 +907,8 @@ public class TestDirectoryScanner { void testScanInfoObject(long blockId, File blockFile, File metaFile) throws Exception { - DirectoryScanner.ScanInfo scanInfo = - new DirectoryScanner.ScanInfo(blockId, blockFile, metaFile, TEST_VOLUME); + FsVolumeSpi.ScanInfo scanInfo = + new FsVolumeSpi.ScanInfo(blockId, blockFile, metaFile, TEST_VOLUME); assertEquals(blockId, scanInfo.getBlockId()); if (blockFile != null) { assertEquals(blockFile.getAbsolutePath(), @@ -906,8 +926,8 @@ public class TestDirectoryScanner { } void testScanInfoObject(long blockId) throws Exception { - DirectoryScanner.ScanInfo scanInfo = - new DirectoryScanner.ScanInfo(blockId, null, null, null); + FsVolumeSpi.ScanInfo scanInfo = + new FsVolumeSpi.ScanInfo(blockId, null, null, null); assertEquals(blockId, scanInfo.getBlockId()); assertNull(scanInfo.getBlockFile()); assertNull(scanInfo.getMetaFile()); @@ -963,8 +983,8 @@ public class TestDirectoryScanner { List volumes = new ArrayList<>(); Iterator iterator = fds.getFsVolumeReferences().iterator(); while (iterator.hasNext()) { - FsVolumeSpi volume = iterator.next(); - FsVolumeSpi spy = Mockito.spy(volume); + FsVolumeImpl volume = (FsVolumeImpl) iterator.next(); + FsVolumeImpl spy = Mockito.spy(volume); Mockito.doThrow(new IOException("Error while getFinalizedDir")) .when(spy).getFinalizedDir(volume.getBlockPoolList()[0]); volumes.add(spy); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java index 86d2ff4c960..2103392db95 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java @@ -199,7 +199,7 @@ public class TestDiskError { try (FsDatasetSpi.FsVolumeReferences volumes = dn.getFSDataset().getFsVolumeReferences()) { for (FsVolumeSpi vol : volumes) { - String dir = vol.getBasePath(); + String dir = vol.getStorageLocation().getFile().getAbsolutePath(); Path dataDir = new Path(dir); FsPermission actual = localFS.getFileStatus(dataDir).getPermission(); assertEquals("Permission for dir: " + dataDir + ", is " + actual + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java index 126810825b5..7b7f04f2a86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java @@ -56,12 +56,14 @@ public class ExternalDatasetImpl implements FsDatasetSpi { } @Override - public void addVolume(StorageLocation location, List nsInfos) throws IOException { - + public void addVolume(StorageLocation location, List nsInfos) + throws IOException { } @Override - public void removeVolumes(Set volumes, boolean clearFailure) { + public void removeVolumes(Collection volumes, + boolean clearFailure) { + throw new UnsupportedOperationException(); } @Override @@ -242,7 +244,7 @@ public class ExternalDatasetImpl implements FsDatasetSpi { } @Override - public Set checkDataDir() { + public Set checkDataDir() { return null; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java index 985a259a07f..83d6c4c6794 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java @@ -18,11 +18,16 @@ package org.apache.hadoop.hdfs.server.datanode.extdataset; -import java.io.File; import java.io.IOException; +import java.net.URI; import java.nio.channels.ClosedChannelException; +import java.util.LinkedList; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.DF; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.server.datanode.StorageLocation; +import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ReportCompiler; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -43,21 +48,6 @@ public class ExternalVolumeImpl implements FsVolumeSpi { return 0; } - @Override - public String getBasePath() { - return null; - } - - @Override - public String getPath(String bpid) throws IOException { - return null; - } - - @Override - public File getFinalizedDir(String bpid) throws IOException { - return null; - } - @Override public String getStorageID() { return null; @@ -100,4 +90,26 @@ public class ExternalVolumeImpl implements FsVolumeSpi { public FsDatasetSpi getDataset() { return null; } + + @Override + public StorageLocation getStorageLocation() { + return null; + } + + @Override + public URI getBaseURI() { + return null; + } + + @Override + public DF getUsageStats(Configuration conf) { + return null; + } + + @Override + public LinkedList compileReport(String bpid, + LinkedList report, ReportCompiler reportCompiler) + throws InterruptedException, IOException { + return null; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java index a465c058e04..07ddb5984b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java @@ -374,9 +374,12 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils { public long getRawCapacity() throws IOException { try (FsVolumeReferences volRefs = dataset.getFsVolumeReferences()) { Preconditions.checkState(volRefs.size() != 0); - DF df = new DF(new File(volRefs.get(0).getBasePath()), - dataset.datanode.getConf()); - return df.getCapacity(); + DF df = volRefs.get(0).getUsageStats(dataset.datanode.getConf()); + if (df != null) { + return df.getCapacity(); + } else { + return -1; + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java index 179b6172117..e48aae0a554 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.Storage; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.datanode.BlockScanner; import org.apache.hadoop.hdfs.server.datanode.DNConf; @@ -50,7 +51,9 @@ import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; import org.apache.hadoop.hdfs.server.datanode.ShortCircuitRegistry; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi.FsVolumeReferences; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.io.MultipleIOException; @@ -122,8 +125,10 @@ public class TestFsDatasetImpl { private final static String BLOCKPOOL = "BP-TEST"; - private static Storage.StorageDirectory createStorageDirectory(File root) { - Storage.StorageDirectory sd = new Storage.StorageDirectory(root); + private static Storage.StorageDirectory createStorageDirectory(File root) + throws SecurityException, IOException { + Storage.StorageDirectory sd = new Storage.StorageDirectory( + StorageLocation.parse(root.toURI().toString())); DataStorage.createStorageID(sd, false); return sd; } @@ -196,16 +201,18 @@ public class TestFsDatasetImpl { for (int i = 0; i < numNewVolumes; i++) { String path = BASE_DIR + "/newData" + i; String pathUri = new Path(path).toUri().toString(); - expectedVolumes.add(new File(pathUri).toString()); + expectedVolumes.add(new File(pathUri).getAbsolutePath()); StorageLocation loc = StorageLocation.parse(pathUri); Storage.StorageDirectory sd = createStorageDirectory(new File(path)); DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd); - when(storage.prepareVolume(eq(datanode), eq(loc.getFile()), + when(storage.prepareVolume(eq(datanode), eq(loc), anyListOf(NamespaceInfo.class))) .thenReturn(builder); dataset.addVolume(loc, nsInfos); + LOG.info("expectedVolumes " + i + " is " + + new File(pathUri).getAbsolutePath()); } assertEquals(totalVolumes, getNumVolumes()); @@ -215,7 +222,9 @@ public class TestFsDatasetImpl { try (FsDatasetSpi.FsVolumeReferences volumes = dataset.getFsVolumeReferences()) { for (int i = 0; i < numNewVolumes; i++) { - actualVolumes.add(volumes.get(numExistingVolumes + i).getBasePath()); + String volumeName = volumes.get(numExistingVolumes + i).toString(); + actualVolumes.add(volumeName); + LOG.info("actualVolume " + i + " is " + volumeName); } } assertEquals(actualVolumes.size(), expectedVolumes.size()); @@ -262,9 +271,18 @@ public class TestFsDatasetImpl { final String[] dataDirs = conf.get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY).split(","); final String volumePathToRemove = dataDirs[0]; - Set volumesToRemove = new HashSet<>(); - volumesToRemove.add(StorageLocation.parse(volumePathToRemove).getFile()); + Set volumesToRemove = new HashSet<>(); + volumesToRemove.add(StorageLocation.parse(volumePathToRemove)); + FsVolumeReferences volReferences = dataset.getFsVolumeReferences(); + FsVolumeImpl volumeToRemove = null; + for (FsVolumeSpi vol: volReferences) { + if (vol.getStorageLocation().equals(volumesToRemove.iterator().next())) { + volumeToRemove = (FsVolumeImpl) vol; + } + } + assertTrue(volumeToRemove != null); + volReferences.close(); dataset.removeVolumes(volumesToRemove, true); int expectedNumVolumes = dataDirs.length - 1; assertEquals("The volume has been removed from the volumeList.", @@ -273,7 +291,7 @@ public class TestFsDatasetImpl { expectedNumVolumes, dataset.storageMap.size()); try { - dataset.asyncDiskService.execute(volumesToRemove.iterator().next(), + dataset.asyncDiskService.execute(volumeToRemove, new Runnable() { @Override public void run() {} @@ -281,7 +299,7 @@ public class TestFsDatasetImpl { fail("Expect RuntimeException: the volume has been removed from the " + "AsyncDiskService."); } catch (RuntimeException e) { - GenericTestUtils.assertExceptionContains("Cannot find root", e); + GenericTestUtils.assertExceptionContains("Cannot find volume", e); } int totalNumReplicas = 0; @@ -306,7 +324,7 @@ public class TestFsDatasetImpl { Storage.StorageDirectory sd = createStorageDirectory(new File(newVolumePath)); DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd); - when(storage.prepareVolume(eq(datanode), eq(loc.getFile()), + when(storage.prepareVolume(eq(datanode), eq(loc), anyListOf(NamespaceInfo.class))) .thenReturn(builder); @@ -315,8 +333,8 @@ public class TestFsDatasetImpl { when(storage.getNumStorageDirs()).thenReturn(numExistingVolumes + 1); when(storage.getStorageDir(numExistingVolumes)).thenReturn(sd); - Set volumesToRemove = new HashSet<>(); - volumesToRemove.add(loc.getFile()); + Set volumesToRemove = new HashSet<>(); + volumesToRemove.add(loc); dataset.removeVolumes(volumesToRemove, true); assertEquals(numExistingVolumes, getNumVolumes()); } @@ -336,7 +354,8 @@ public class TestFsDatasetImpl { for (int i = 0; i < NUM_VOLUMES; i++) { FsVolumeImpl volume = mock(FsVolumeImpl.class); oldVolumes.add(volume); - when(volume.getBasePath()).thenReturn("data" + i); + when(volume.getStorageLocation()).thenReturn( + StorageLocation.parse(new File("data" + i).toURI().toString())); when(volume.checkClosed()).thenReturn(true); FsVolumeReference ref = mock(FsVolumeReference.class); when(ref.getVolume()).thenReturn(volume); @@ -348,13 +367,16 @@ public class TestFsDatasetImpl { final FsVolumeImpl newVolume = mock(FsVolumeImpl.class); final FsVolumeReference newRef = mock(FsVolumeReference.class); when(newRef.getVolume()).thenReturn(newVolume); - when(newVolume.getBasePath()).thenReturn("data4"); + when(newVolume.getStorageLocation()).thenReturn( + StorageLocation.parse(new File("data4").toURI().toString())); FsVolumeImpl blockedVolume = volumeList.getVolumes().get(1); doAnswer(new Answer() { @Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - volumeList.removeVolume(new File("data4"), false); + volumeList.removeVolume( + StorageLocation.parse((new File("data4")).toURI().toString()), + false); volumeList.addVolume(newRef); return null; } @@ -386,7 +408,8 @@ public class TestFsDatasetImpl { File badDir = new File(BASE_DIR, "bad"); badDir.mkdirs(); doReturn(mockVolume).when(spyDataset) - .createFsVolume(anyString(), any(File.class), any(StorageType.class)); + .createFsVolume(anyString(), any(StorageDirectory.class), + any(StorageLocation.class)); doThrow(new IOException("Failed to getVolumeMap()")) .when(mockVolume).getVolumeMap( anyString(), @@ -396,7 +419,8 @@ public class TestFsDatasetImpl { Storage.StorageDirectory sd = createStorageDirectory(badDir); sd.lock(); DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd); - when(storage.prepareVolume(eq(datanode), eq(badDir.getAbsoluteFile()), + when(storage.prepareVolume(eq(datanode), + eq(StorageLocation.parse(badDir.toURI().toString())), Matchers.>any())) .thenReturn(builder); @@ -540,7 +564,7 @@ public class TestFsDatasetImpl { DataStorage.VolumeBuilder builder = new DataStorage.VolumeBuilder(storage, sd); when( - storage.prepareVolume(eq(datanode), eq(loc.getFile()), + storage.prepareVolume(eq(datanode), eq(loc), anyListOf(NamespaceInfo.class))).thenReturn(builder); String cacheFilePath = @@ -584,7 +608,7 @@ public class TestFsDatasetImpl { return dfsUsed; } - @Test(timeout = 30000) + @Test(timeout = 60000) public void testRemoveVolumeBeingWritten() throws Exception { // Will write and remove on dn0. final ExtendedBlock eb = new ExtendedBlock(BLOCK_POOL_IDS[0], 0); @@ -636,10 +660,9 @@ public class TestFsDatasetImpl { class VolRemoveThread extends Thread { public void run() { - Set volumesToRemove = new HashSet<>(); + Set volumesToRemove = new HashSet<>(); try { - volumesToRemove.add(StorageLocation.parse( - dataset.getVolume(eb).getBasePath()).getFile()); + volumesToRemove.add(dataset.getVolume(eb).getStorageLocation()); } catch (Exception e) { LOG.info("Problem preparing volumes to remove: ", e); Assert.fail("Exception in remove volume thread, check log for " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java index 3d4c38cd938..6eff300febe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java @@ -22,7 +22,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.datanode.BlockScanner; +import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy; import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy; @@ -71,8 +73,13 @@ public class TestFsVolumeList { for (int i = 0; i < 3; i++) { File curDir = new File(baseDir, "nextvolume-" + i); curDir.mkdirs(); - FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", curDir, - conf, StorageType.DEFAULT); + FsVolumeImpl volume = new FsVolumeImplBuilder() + .setConf(conf) + .setDataset(dataset) + .setStorageID("storage-id") + .setStorageDirectory( + new StorageDirectory(StorageLocation.parse(curDir.getPath()))) + .build(); volume.setCapacityForTesting(1024 * 1024 * 1024); volumes.add(volume); volumeList.addVolume(volume.obtainReference()); @@ -109,8 +116,13 @@ public class TestFsVolumeList { for (int i = 0; i < 3; i++) { File curDir = new File(baseDir, "volume-" + i); curDir.mkdirs(); - FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", curDir, - conf, StorageType.DEFAULT); + FsVolumeImpl volume = new FsVolumeImplBuilder() + .setConf(conf) + .setDataset(dataset) + .setStorageID("storage-id") + .setStorageDirectory( + new StorageDirectory(StorageLocation.parse(curDir.getPath()))) + .build(); volumes.add(volume); volumeList.addVolume(volume.obtainReference()); } @@ -139,8 +151,13 @@ public class TestFsVolumeList { Collections.emptyList(), null, blockChooser); File volDir = new File(baseDir, "volume-0"); volDir.mkdirs(); - FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir, - conf, StorageType.DEFAULT); + FsVolumeImpl volume = new FsVolumeImplBuilder() + .setConf(conf) + .setDataset(dataset) + .setStorageID("storage-id") + .setStorageDirectory( + new StorageDirectory(StorageLocation.parse(volDir.getPath()))) + .build(); FsVolumeReference ref = volume.obtainReference(); volumeList.addVolume(ref); assertNull(ref.getVolume()); @@ -155,8 +172,13 @@ public class TestFsVolumeList { volDir.mkdirs(); // when storage type reserved is not configured,should consider // dfs.datanode.du.reserved. - FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir, conf, - StorageType.RAM_DISK); + FsVolumeImpl volume = new FsVolumeImplBuilder().setDataset(dataset) + .setStorageDirectory( + new StorageDirectory( + StorageLocation.parse("[RAM_DISK]"+volDir.getPath()))) + .setStorageID("storage-id") + .setConf(conf) + .build(); assertEquals("", 100L, volume.getReserved()); // when storage type reserved is configured. conf.setLong( @@ -165,17 +187,37 @@ public class TestFsVolumeList { conf.setLong( DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY + "." + StringUtils.toLowerCase(StorageType.SSD.toString()), 2L); - FsVolumeImpl volume1 = new FsVolumeImpl(dataset, "storage-id", volDir, - conf, StorageType.RAM_DISK); + FsVolumeImpl volume1 = new FsVolumeImplBuilder().setDataset(dataset) + .setStorageDirectory( + new StorageDirectory( + StorageLocation.parse("[RAM_DISK]"+volDir.getPath()))) + .setStorageID("storage-id") + .setConf(conf) + .build(); assertEquals("", 1L, volume1.getReserved()); - FsVolumeImpl volume2 = new FsVolumeImpl(dataset, "storage-id", volDir, - conf, StorageType.SSD); + FsVolumeImpl volume2 = new FsVolumeImplBuilder().setDataset(dataset) + .setStorageDirectory( + new StorageDirectory( + StorageLocation.parse("[SSD]"+volDir.getPath()))) + .setStorageID("storage-id") + .setConf(conf) + .build(); assertEquals("", 2L, volume2.getReserved()); - FsVolumeImpl volume3 = new FsVolumeImpl(dataset, "storage-id", volDir, - conf, StorageType.DISK); + FsVolumeImpl volume3 = new FsVolumeImplBuilder().setDataset(dataset) + .setStorageDirectory( + new StorageDirectory( + StorageLocation.parse("[DISK]"+volDir.getPath()))) + .setStorageID("storage-id") + .setConf(conf) + .build(); assertEquals("", 100L, volume3.getReserved()); - FsVolumeImpl volume4 = new FsVolumeImpl(dataset, "storage-id", volDir, - conf, StorageType.DEFAULT); + FsVolumeImpl volume4 = new FsVolumeImplBuilder().setDataset(dataset) + .setStorageDirectory( + new StorageDirectory( + StorageLocation.parse(volDir.getPath()))) + .setStorageID("storage-id") + .setConf(conf) + .build(); assertEquals("", 100L, volume4.getReserved()); } @@ -197,8 +239,13 @@ public class TestFsVolumeList { long actualNonDfsUsage = 300L; long reservedForReplicas = 50L; conf.setLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY, duReserved); - FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir, conf, - StorageType.DEFAULT); + FsVolumeImpl volume = new FsVolumeImplBuilder().setDataset(dataset) + .setStorageDirectory( + new StorageDirectory( + StorageLocation.parse(volDir.getPath()))) + .setStorageID("storage-id") + .setConf(conf) + .build(); FsVolumeImpl spyVolume = Mockito.spy(volume); // Set Capacity for testing long testCapacity = diskCapacity - duReserved; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java index d911e749eaf..9985210f249 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ClusterConnector; import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ConnectorFactory; import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster; import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerDataNode; -import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolume; import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; @@ -137,6 +136,7 @@ public class TestDiskBalancer { final int dataNodeCount = 1; final int dataNodeIndex = 0; final int sourceDiskIndex = 0; + final long cap = blockSize * 2L * blockCount; MiniDFSCluster cluster = new ClusterBuilder() .setBlockCount(blockCount) @@ -144,6 +144,7 @@ public class TestDiskBalancer { .setDiskCount(diskCount) .setNumDatanodes(dataNodeCount) .setConf(conf) + .setCapacities(new long[] {cap, cap}) .build(); try { DataMover dataMover = new DataMover(cluster, dataNodeIndex, @@ -174,7 +175,7 @@ public class TestDiskBalancer { final int dataNodeCount = 1; final int dataNodeIndex = 0; final int sourceDiskIndex = 0; - + final long cap = blockSize * 2L * blockCount; MiniDFSCluster cluster = new ClusterBuilder() .setBlockCount(blockCount) @@ -182,9 +183,9 @@ public class TestDiskBalancer { .setDiskCount(diskCount) .setNumDatanodes(dataNodeCount) .setConf(conf) + .setCapacities(new long[] {cap, cap, cap}) .build(); - try { DataMover dataMover = new DataMover(cluster, dataNodeIndex, sourceDiskIndex, conf, blockSize, blockCount); @@ -221,6 +222,7 @@ public class TestDiskBalancer { final int dataNodeCount = 1; final int dataNodeIndex = 0; final int sourceDiskIndex = 0; + final long cap = blockSize * 2L * blockCount; MiniDFSCluster cluster = new ClusterBuilder() .setBlockCount(blockCount) @@ -228,6 +230,7 @@ public class TestDiskBalancer { .setDiskCount(diskCount) .setNumDatanodes(dataNodeCount) .setConf(conf) + .setCapacities(new long[] {cap, cap}) .build(); try { @@ -245,24 +248,6 @@ public class TestDiskBalancer { } } - /** - * Sets alll Disks capacity to size specified. - * - * @param cluster - DiskBalancerCluster - * @param size - new size of the disk - */ - private void setVolumeCapacity(DiskBalancerCluster cluster, long size, - String diskType) { - Preconditions.checkNotNull(cluster); - for (DiskBalancerDataNode node : cluster.getNodes()) { - for (DiskBalancerVolume vol : - node.getVolumeSets().get(diskType).getVolumes()) { - vol.setCapacity(size); - } - node.getVolumeSets().get(diskType).computeVolumeDataDensity(); - } - } - /** * Helper class that allows us to create different kinds of MiniDFSClusters * and populate data. @@ -274,6 +259,7 @@ public class TestDiskBalancer { private int fileLen; private int blockCount; private int diskCount; + private long[] capacities; public ClusterBuilder setConf(Configuration conf) { this.conf = conf; @@ -300,13 +286,9 @@ public class TestDiskBalancer { return this; } - private long[] getCapacities(int diskCount, int bSize, int fSize) { - Preconditions.checkState(diskCount > 0); - long[] capacities = new long[diskCount]; - for (int x = 0; x < diskCount; x++) { - capacities[x] = diskCount * bSize * fSize * 2L; - } - return capacities; + private ClusterBuilder setCapacities(final long[] caps) { + this.capacities = caps; + return this; } private StorageType[] getStorageTypes(int diskCount) { @@ -338,7 +320,7 @@ public class TestDiskBalancer { // Write a file and restart the cluster MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numDatanodes) - .storageCapacities(getCapacities(diskCount, blockSize, fileLen)) + .storageCapacities(capacities) .storageTypes(getStorageTypes(diskCount)) .storagesPerDatanode(diskCount) .build(); @@ -447,10 +429,6 @@ public class TestDiskBalancer { diskBalancerCluster.readClusterInfo(); List nodesToProcess = new LinkedList<>(); - // Rewrite the capacity in the model to show that disks need - // re-balancing. - setVolumeCapacity(diskBalancerCluster, blockSize * 2L * blockCount, - "DISK"); // Pick a node to process. nodesToProcess.add(diskBalancerCluster.getNodeByUUID( node.getDatanodeUuid())); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java index 794a887aa68..7df03333b4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java @@ -331,8 +331,8 @@ public class TestDiskBalancerWithMockMover { .getFsVolumeReferences(); nodeID = dataNode.getDatanodeUuid(); - sourceName = references.get(0).getBasePath(); - destName = references.get(1).getBasePath(); + sourceName = references.get(0).getBaseURI().getPath(); + destName = references.get(1).getBaseURI().getPath(); sourceUUID = references.get(0).getStorageID(); destUUID = references.get(1).getStorageID(); references.close(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index efb5cf8faf7..658e4ca3dd7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -72,6 +72,7 @@ import org.apache.hadoop.hdfs.protocol.CachePoolStats; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList.Type; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; @@ -89,6 +90,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import com.google.common.base.Supplier; @@ -1531,4 +1533,12 @@ public class TestCacheDirectives { DataNodeTestUtils.setCacheReportsDisabledForTests(cluster, false); } } + + @Test + public void testNoLookupsWhenNotUsed() throws Exception { + CacheManager cm = cluster.getNamesystem().getCacheManager(); + LocatedBlocks locations = Mockito.mock(LocatedBlocks.class); + cm.setCachedLocations(locations); + Mockito.verifyZeroInteractions(locations); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java index 2b43c0f9536..071bdf7e4a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.BufferedReader; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.StringReader; import java.util.EnumSet; @@ -30,6 +31,7 @@ import com.google.common.collect.ImmutableList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.XAttrSetFlag; @@ -386,4 +388,50 @@ public class TestFSDirectory { XAttrSetFlag.REPLACE)); verifyXAttrsPresent(newXAttrs, 4); } + + @Test + public void testVerifyParentDir() throws Exception { + hdfs.mkdirs(new Path("/dir1/dir2")); + hdfs.createNewFile(new Path("/dir1/file")); + hdfs.createNewFile(new Path("/dir1/dir2/file")); + + INodesInPath iip = fsdir.resolvePath(null, "/"); + fsdir.verifyParentDir(iip); + + iip = fsdir.resolvePath(null, "/dir1"); + fsdir.verifyParentDir(iip); + + iip = fsdir.resolvePath(null, "/dir1/file"); + fsdir.verifyParentDir(iip); + + iip = fsdir.resolvePath(null, "/dir-nonexist/file"); + try { + fsdir.verifyParentDir(iip); + fail("expected FNF"); + } catch (FileNotFoundException fnf) { + // expected. + } + + iip = fsdir.resolvePath(null, "/dir1/dir2"); + fsdir.verifyParentDir(iip); + + iip = fsdir.resolvePath(null, "/dir1/dir2/file"); + fsdir.verifyParentDir(iip); + + iip = fsdir.resolvePath(null, "/dir1/dir-nonexist/file"); + try { + fsdir.verifyParentDir(iip); + fail("expected FNF"); + } catch (FileNotFoundException fnf) { + // expected. + } + + iip = fsdir.resolvePath(null, "/dir1/file/fail"); + try { + fsdir.verifyParentDir(iip); + fail("expected FNF"); + } catch (ParentNotDirectoryException pnd) { + // expected. + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java index 47d549b1af4..f02c679f388 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; -import org.apache.hadoop.util.FakeTimer; import static org.hamcrest.CoreMatchers.either; import static org.hamcrest.CoreMatchers.instanceOf; import static org.junit.Assert.*; @@ -31,7 +30,6 @@ import java.net.InetAddress; import java.net.URI; import java.util.Collection; -import com.google.common.base.Supplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileUtil; @@ -45,22 +43,12 @@ import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; import org.apache.hadoop.hdfs.server.namenode.ha.HAState; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.test.GenericTestUtils.LogCapturer; -import org.apache.log4j.Level; import org.junit.After; -import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; import org.mockito.internal.util.reflection.Whitebox; import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.regex.Pattern; public class TestFSNamesystem { @@ -165,59 +153,6 @@ public class TestFSNamesystem { assertTrue("Replication queues weren't being populated after entering " + "safemode 2nd time", bm.isPopulatingReplQueues()); } - - @Test - public void testFsLockFairness() throws IOException, InterruptedException{ - Configuration conf = new Configuration(); - - FSEditLog fsEditLog = Mockito.mock(FSEditLog.class); - FSImage fsImage = Mockito.mock(FSImage.class); - Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog); - - conf.setBoolean("dfs.namenode.fslock.fair", true); - FSNamesystem fsNamesystem = new FSNamesystem(conf, fsImage); - assertTrue(fsNamesystem.getFsLockForTests().isFair()); - - conf.setBoolean("dfs.namenode.fslock.fair", false); - fsNamesystem = new FSNamesystem(conf, fsImage); - assertFalse(fsNamesystem.getFsLockForTests().isFair()); - } - - @Test - public void testFSNamesystemLockCompatibility() { - FSNamesystemLock rwLock = new FSNamesystemLock(true); - - assertEquals(0, rwLock.getReadHoldCount()); - rwLock.readLock().lock(); - assertEquals(1, rwLock.getReadHoldCount()); - - rwLock.readLock().lock(); - assertEquals(2, rwLock.getReadHoldCount()); - - rwLock.readLock().unlock(); - assertEquals(1, rwLock.getReadHoldCount()); - - rwLock.readLock().unlock(); - assertEquals(0, rwLock.getReadHoldCount()); - - assertFalse(rwLock.isWriteLockedByCurrentThread()); - assertEquals(0, rwLock.getWriteHoldCount()); - rwLock.writeLock().lock(); - assertTrue(rwLock.isWriteLockedByCurrentThread()); - assertEquals(1, rwLock.getWriteHoldCount()); - - rwLock.writeLock().lock(); - assertTrue(rwLock.isWriteLockedByCurrentThread()); - assertEquals(2, rwLock.getWriteHoldCount()); - - rwLock.writeLock().unlock(); - assertTrue(rwLock.isWriteLockedByCurrentThread()); - assertEquals(1, rwLock.getWriteHoldCount()); - - rwLock.writeLock().unlock(); - assertFalse(rwLock.isWriteLockedByCurrentThread()); - assertEquals(0, rwLock.getWriteHoldCount()); - } @Test public void testReset() throws Exception { @@ -257,233 +192,6 @@ public class TestFSNamesystem { FSNamesystem.getEffectiveLayoutVersion(false, -63, -61, -63)); } - @Test - public void testFSLockGetWaiterCount() throws InterruptedException { - final int threadCount = 3; - final CountDownLatch latch = new CountDownLatch(threadCount); - final FSNamesystemLock rwLock = new FSNamesystemLock(true); - rwLock.writeLock().lock(); - ExecutorService helper = Executors.newFixedThreadPool(threadCount); - - for (int x = 0; x < threadCount; x++) { - helper.execute(new Runnable() { - @Override - public void run() { - latch.countDown(); - rwLock.readLock().lock(); - } - }); - } - - latch.await(); - try { - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - return (threadCount == rwLock.getQueueLength()); - } - }, 10, 1000); - } catch (TimeoutException e) { - fail("Expected number of blocked thread not found"); - } - } - - /** - * Test when FSNamesystem write lock is held for a long time, - * logger will report it. - */ - @Test(timeout=45000) - public void testFSWriteLockLongHoldingReport() throws Exception { - final long writeLockReportingThreshold = 100L; - final long writeLockSuppressWarningInterval = 10000L; - Configuration conf = new Configuration(); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY, - writeLockReportingThreshold); - conf.setTimeDuration(DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY, - writeLockSuppressWarningInterval, TimeUnit.MILLISECONDS); - FSImage fsImage = Mockito.mock(FSImage.class); - FSEditLog fsEditLog = Mockito.mock(FSEditLog.class); - Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog); - FSNamesystem fsn = new FSNamesystem(conf, fsImage); - - FakeTimer timer = new FakeTimer(); - fsn.setTimer(timer); - timer.advance(writeLockSuppressWarningInterval); - - LogCapturer logs = LogCapturer.captureLogs(FSNamesystem.LOG); - GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.INFO); - - // Don't report if the write lock is held for a short time - fsn.writeLock(); - fsn.writeUnlock(); - assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); - - // Report the first write lock warning if it is held for a long time - fsn.writeLock(); - timer.advance(writeLockReportingThreshold + 10); - logs.clearOutput(); - fsn.writeUnlock(); - assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName())); - - // Track but do not Report if the write lock is held (interruptibly) for - // a long time but time since last report does not exceed the suppress - // warning interval - fsn.writeLockInterruptibly(); - timer.advance(writeLockReportingThreshold + 10); - logs.clearOutput(); - fsn.writeUnlock(); - assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); - - // Track but do not Report if it's held for a long time when re-entering - // write lock but time since last report does not exceed the suppress - // warning interval - fsn.writeLock(); - timer.advance(writeLockReportingThreshold/ 2 + 1); - fsn.writeLockInterruptibly(); - timer.advance(writeLockReportingThreshold/ 2 + 1); - fsn.writeLock(); - timer.advance(writeLockReportingThreshold/ 2); - logs.clearOutput(); - fsn.writeUnlock(); - assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); - logs.clearOutput(); - fsn.writeUnlock(); - assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); - logs.clearOutput(); - fsn.writeUnlock(); - assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); - - // Report if it's held for a long time and time since last report exceeds - // the supress warning interval - timer.advance(writeLockSuppressWarningInterval); - fsn.writeLock(); - timer.advance(writeLockReportingThreshold + 100); - logs.clearOutput(); - fsn.writeUnlock(); - assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName())); - assertTrue(logs.getOutput().contains("Number of suppressed write-lock " + - "reports: 2")); - } - - /** - * Test when FSNamesystem read lock is held for a long time, - * logger will report it. - */ - @Test(timeout=45000) - public void testFSReadLockLongHoldingReport() throws Exception { - final long readLockReportingThreshold = 100L; - final long readLockSuppressWarningInterval = 10000L; - final String readLockLogStmt = "FSNamesystem read lock held for "; - Configuration conf = new Configuration(); - conf.setLong( - DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY, - readLockReportingThreshold); - conf.setTimeDuration(DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY, - readLockSuppressWarningInterval, TimeUnit.MILLISECONDS); - FSImage fsImage = Mockito.mock(FSImage.class); - FSEditLog fsEditLog = Mockito.mock(FSEditLog.class); - Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog); - FSNamesystem fsn = new FSNamesystem(conf, fsImage); - - FakeTimer timer = new FakeTimer(); - fsn.setTimer(timer); - timer.advance(readLockSuppressWarningInterval); - - LogCapturer logs = LogCapturer.captureLogs(FSNamesystem.LOG); - GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.INFO); - - // Don't report if the read lock is held for a short time - fsn.readLock(); - fsn.readUnlock(); - assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()) && - logs.getOutput().contains(readLockLogStmt)); - - // Report the first read lock warning if it is held for a long time - fsn.readLock(); - timer.advance(readLockReportingThreshold + 10); - logs.clearOutput(); - fsn.readUnlock(); - assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName()) - && logs.getOutput().contains(readLockLogStmt)); - - // Track but do not Report if the write lock is held for a long time but - // time since last report does not exceed the suppress warning interval - fsn.readLock(); - timer.advance(readLockReportingThreshold + 10); - logs.clearOutput(); - fsn.readUnlock(); - assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()) - && logs.getOutput().contains(readLockLogStmt)); - - // Track but do not Report if it's held for a long time when re-entering - // read lock but time since last report does not exceed the suppress - // warning interval - fsn.readLock(); - timer.advance(readLockReportingThreshold / 2 + 1); - fsn.readLock(); - timer.advance(readLockReportingThreshold / 2 + 1); - logs.clearOutput(); - fsn.readUnlock(); - assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()) || - logs.getOutput().contains(readLockLogStmt)); - logs.clearOutput(); - fsn.readUnlock(); - assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()) && - logs.getOutput().contains(readLockLogStmt)); - - // Report if it's held for a long time (and time since last report - // exceeds the suppress warning interval) while another thread also has the - // read lock. Let one thread hold the lock long enough to activate an - // alert, then have another thread grab the read lock to ensure that this - // doesn't reset the timing. - timer.advance(readLockSuppressWarningInterval); - logs.clearOutput(); - CountDownLatch barrier = new CountDownLatch(1); - CountDownLatch barrier2 = new CountDownLatch(1); - Thread t1 = new Thread() { - @Override - public void run() { - try { - fsn.readLock(); - timer.advance(readLockReportingThreshold + 1); - barrier.countDown(); // Allow for t2 to acquire the read lock - barrier2.await(); // Wait until t2 has the read lock - fsn.readUnlock(); - } catch (InterruptedException e) { - fail("Interrupted during testing"); - } - } - }; - Thread t2 = new Thread() { - @Override - public void run() { - try { - barrier.await(); // Wait until t1 finishes sleeping - fsn.readLock(); - barrier2.countDown(); // Allow for t1 to unlock - fsn.readUnlock(); - } catch (InterruptedException e) { - fail("Interrupted during testing"); - } - } - }; - t1.start(); - t2.start(); - t1.join(); - t2.join(); - // Look for the differentiating class names in the stack trace - String stackTracePatternString = - String.format("INFO.+%s(.+\n){4}\\Q%%s\\E\\.run", readLockLogStmt); - Pattern t1Pattern = Pattern.compile( - String.format(stackTracePatternString, t1.getClass().getName())); - assertTrue(t1Pattern.matcher(logs.getOutput()).find()); - Pattern t2Pattern = Pattern.compile( - String.format(stackTracePatternString, t2.getClass().getName())); - assertFalse(t2Pattern.matcher(logs.getOutput()).find()); - assertTrue(logs.getOutput().contains("Number of suppressed read-lock " + - "reports: 2")); - } - @Test public void testSafemodeReplicationConf() throws IOException { Configuration conf = new Configuration(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java new file mode 100644 index 00000000000..08900ecba8f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java @@ -0,0 +1,317 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + +import com.google.common.base.Supplier; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.GenericTestUtils.LogCapturer; +import org.apache.hadoop.util.FakeTimer; +import org.apache.log4j.Level; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; + +import static org.junit.Assert.*; + +/** + * Tests the FSNamesystemLock, looking at lock compatibilities and + * proper logging of lock hold times. + */ +public class TestFSNamesystemLock { + + @Test + public void testFsLockFairness() throws IOException, InterruptedException{ + Configuration conf = new Configuration(); + + conf.setBoolean("dfs.namenode.fslock.fair", true); + FSNamesystemLock fsnLock = new FSNamesystemLock(conf); + assertTrue(fsnLock.coarseLock.isFair()); + + conf.setBoolean("dfs.namenode.fslock.fair", false); + fsnLock = new FSNamesystemLock(conf); + assertFalse(fsnLock.coarseLock.isFair()); + } + + @Test + public void testFSNamesystemLockCompatibility() { + FSNamesystemLock rwLock = new FSNamesystemLock(new Configuration()); + + assertEquals(0, rwLock.getReadHoldCount()); + rwLock.readLock(); + assertEquals(1, rwLock.getReadHoldCount()); + + rwLock.readLock(); + assertEquals(2, rwLock.getReadHoldCount()); + + rwLock.readUnlock(); + assertEquals(1, rwLock.getReadHoldCount()); + + rwLock.readUnlock(); + assertEquals(0, rwLock.getReadHoldCount()); + + assertFalse(rwLock.isWriteLockedByCurrentThread()); + assertEquals(0, rwLock.getWriteHoldCount()); + rwLock.writeLock(); + assertTrue(rwLock.isWriteLockedByCurrentThread()); + assertEquals(1, rwLock.getWriteHoldCount()); + + rwLock.writeLock(); + assertTrue(rwLock.isWriteLockedByCurrentThread()); + assertEquals(2, rwLock.getWriteHoldCount()); + + rwLock.writeUnlock(); + assertTrue(rwLock.isWriteLockedByCurrentThread()); + assertEquals(1, rwLock.getWriteHoldCount()); + + rwLock.writeUnlock(); + assertFalse(rwLock.isWriteLockedByCurrentThread()); + assertEquals(0, rwLock.getWriteHoldCount()); + } + + @Test + public void testFSLockGetWaiterCount() throws InterruptedException { + final int threadCount = 3; + final CountDownLatch latch = new CountDownLatch(threadCount); + final Configuration conf = new Configuration(); + conf.setBoolean("dfs.namenode.fslock.fair", true); + final FSNamesystemLock rwLock = new FSNamesystemLock(conf); + rwLock.writeLock(); + ExecutorService helper = Executors.newFixedThreadPool(threadCount); + + for (int x = 0; x < threadCount; x++) { + helper.execute(new Runnable() { + @Override + public void run() { + latch.countDown(); + rwLock.readLock(); + } + }); + } + + latch.await(); + try { + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return (threadCount == rwLock.getQueueLength()); + } + }, 10, 1000); + } catch (TimeoutException e) { + fail("Expected number of blocked thread not found"); + } + } + + /** + * Test when FSNamesystem write lock is held for a long time, + * logger will report it. + */ + @Test(timeout=45000) + public void testFSWriteLockLongHoldingReport() throws Exception { + final long writeLockReportingThreshold = 100L; + final long writeLockSuppressWarningInterval = 10000L; + Configuration conf = new Configuration(); + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY, + writeLockReportingThreshold); + conf.setTimeDuration(DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY, + writeLockSuppressWarningInterval, TimeUnit.MILLISECONDS); + + final FakeTimer timer = new FakeTimer(); + final FSNamesystemLock fsnLock = new FSNamesystemLock(conf, timer); + timer.advance(writeLockSuppressWarningInterval); + + LogCapturer logs = LogCapturer.captureLogs(FSNamesystem.LOG); + GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.INFO); + + // Don't report if the write lock is held for a short time + fsnLock.writeLock(); + fsnLock.writeUnlock(); + assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); + + // Report if the write lock is held for a long time + fsnLock.writeLock(); + timer.advance(writeLockReportingThreshold + 10); + logs.clearOutput(); + fsnLock.writeUnlock(); + assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName())); + + // Track but do not report if the write lock is held (interruptibly) for + // a long time but time since last report does not exceed the suppress + // warning interval + fsnLock.writeLockInterruptibly(); + timer.advance(writeLockReportingThreshold + 10); + logs.clearOutput(); + fsnLock.writeUnlock(); + assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); + + // Track but do not report if it's held for a long time when re-entering + // write lock but time since last report does not exceed the suppress + // warning interval + fsnLock.writeLock(); + timer.advance(writeLockReportingThreshold / 2 + 1); + fsnLock.writeLockInterruptibly(); + timer.advance(writeLockReportingThreshold / 2 + 1); + fsnLock.writeLock(); + timer.advance(writeLockReportingThreshold / 2); + logs.clearOutput(); + fsnLock.writeUnlock(); + assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); + logs.clearOutput(); + fsnLock.writeUnlock(); + assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); + logs.clearOutput(); + fsnLock.writeUnlock(); + assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); + + // Report if it's held for a long time and time since last report exceeds + // the supress warning interval + timer.advance(writeLockSuppressWarningInterval); + fsnLock.writeLock(); + timer.advance(writeLockReportingThreshold + 100); + logs.clearOutput(); + fsnLock.writeUnlock(); + assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName())); + assertTrue(logs.getOutput().contains( + "Number of suppressed write-lock reports: 2")); + } + + /** + * Test when FSNamesystem read lock is held for a long time, + * logger will report it. + */ + @Test(timeout=45000) + public void testFSReadLockLongHoldingReport() throws Exception { + final long readLockReportingThreshold = 100L; + final long readLockSuppressWarningInterval = 10000L; + final String readLockLogStmt = "FSNamesystem read lock held for "; + Configuration conf = new Configuration(); + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY, + readLockReportingThreshold); + conf.setTimeDuration(DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY, + readLockSuppressWarningInterval, TimeUnit.MILLISECONDS); + + final FakeTimer timer = new FakeTimer(); + final FSNamesystemLock fsnLock = new FSNamesystemLock(conf, timer); + timer.advance(readLockSuppressWarningInterval); + + LogCapturer logs = LogCapturer.captureLogs(FSNamesystem.LOG); + GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.INFO); + + // Don't report if the read lock is held for a short time + fsnLock.readLock(); + fsnLock.readUnlock(); + assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()) && + logs.getOutput().contains(readLockLogStmt)); + + // Report the first read lock warning if it is held for a long time + fsnLock.readLock(); + timer.advance(readLockReportingThreshold + 10); + logs.clearOutput(); + fsnLock.readUnlock(); + assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName()) && + logs.getOutput().contains(readLockLogStmt)); + + // Track but do not Report if the write lock is held for a long time but + // time since last report does not exceed the suppress warning interval + fsnLock.readLock(); + timer.advance(readLockReportingThreshold + 10); + logs.clearOutput(); + fsnLock.readUnlock(); + assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()) && + logs.getOutput().contains(readLockLogStmt)); + + // Track but do not Report if it's held for a long time when re-entering + // read lock but time since last report does not exceed the suppress + // warning interval + fsnLock.readLock(); + timer.advance(readLockReportingThreshold / 2 + 1); + fsnLock.readLock(); + timer.advance(readLockReportingThreshold / 2 + 1); + logs.clearOutput(); + fsnLock.readUnlock(); + assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()) || + logs.getOutput().contains(readLockLogStmt)); + logs.clearOutput(); + fsnLock.readUnlock(); + assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()) && + logs.getOutput().contains(readLockLogStmt)); + + // Report if it's held for a long time (and time since last report + // exceeds the suppress warning interval) while another thread also has the + // read lock. Let one thread hold the lock long enough to activate an + // alert, then have another thread grab the read lock to ensure that this + // doesn't reset the timing. + timer.advance(readLockSuppressWarningInterval); + logs.clearOutput(); + final CountDownLatch barrier = new CountDownLatch(1); + final CountDownLatch barrier2 = new CountDownLatch(1); + Thread t1 = new Thread() { + @Override + public void run() { + try { + fsnLock.readLock(); + timer.advance(readLockReportingThreshold + 1); + barrier.countDown(); // Allow for t2 to acquire the read lock + barrier2.await(); // Wait until t2 has the read lock + fsnLock.readUnlock(); + } catch (InterruptedException e) { + fail("Interrupted during testing"); + } + } + }; + Thread t2 = new Thread() { + @Override + public void run() { + try { + barrier.await(); // Wait until t1 finishes sleeping + fsnLock.readLock(); + barrier2.countDown(); // Allow for t1 to unlock + fsnLock.readUnlock(); + } catch (InterruptedException e) { + fail("Interrupted during testing"); + } + } + }; + t1.start(); + t2.start(); + t1.join(); + t2.join(); + // Look for the differentiating class names in the stack trace + String stackTracePatternString = + String.format("INFO.+%s(.+\n){4}\\Q%%s\\E\\.run", readLockLogStmt); + Pattern t1Pattern = Pattern.compile( + String.format(stackTracePatternString, t1.getClass().getName())); + assertTrue(t1Pattern.matcher(logs.getOutput()).find()); + Pattern t2Pattern = Pattern.compile( + String.format(stackTracePatternString, t2.getClass().getName())); + assertFalse(t2Pattern.matcher(logs.getOutput()).find()); + assertTrue(logs.getOutput().contains( + "Number of suppressed read-lock reports: 2")); + } + +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 9fb4fe57657..aa41e9b0924 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -52,12 +52,16 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.concurrent.TimeoutException; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.base.Supplier; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ChecksumException; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; @@ -73,9 +77,9 @@ import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; @@ -87,6 +91,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; @@ -94,12 +99,15 @@ import org.apache.hadoop.hdfs.server.blockmanagement.CombinedHostFileManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.blockmanagement.HostConfigManager; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck.Result; import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck.ReplicationResult; import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck.ErasureCodingResult; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.tools.DFSck; import org.apache.hadoop.hdfs.util.HostsFileWriter; +import org.apache.hadoop.hdfs.util.StripedBlockUtil; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.AccessControlException; @@ -110,44 +118,49 @@ import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.log4j.PatternLayout; import org.apache.log4j.RollingFileAppender; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import com.google.common.collect.Sets; /** - * A JUnit test for doing fsck + * A JUnit test for doing fsck. */ public class TestFsck { + private static final Log LOG = + LogFactory.getLog(TestFsck.class.getName()); + static final String AUDITLOG_FILE = GenericTestUtils.getTempPath("TestFsck-audit.log"); // Pattern for: // allowed=true ugi=name ip=/address cmd=FSCK src=/ dst=null perm=null - static final Pattern fsckPattern = Pattern.compile( + static final Pattern FSCK_PATTERN = Pattern.compile( "allowed=.*?\\s" + "ugi=.*?\\s" + "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + "cmd=fsck\\ssrc=\\/\\sdst=null\\s" + "perm=null\\s" + "proto=.*"); - static final Pattern getfileinfoPattern = Pattern.compile( + static final Pattern GET_FILE_INFO_PATTERN = Pattern.compile( "allowed=.*?\\s" + "ugi=.*?\\s" + "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + "cmd=getfileinfo\\ssrc=\\/\\sdst=null\\s" + "perm=null\\s" + "proto=.*"); - static final Pattern numMissingBlocksPattern = Pattern.compile( + static final Pattern NUM_MISSING_BLOCKS_PATTERN = Pattern.compile( ".*Missing blocks:\t\t([0123456789]*).*"); - static final Pattern numCorruptBlocksPattern = Pattern.compile( + static final Pattern NUM_CORRUPT_BLOCKS_PATTERN = Pattern.compile( ".*Corrupt blocks:\t\t([0123456789]*).*"); private static final String LINE_SEPARATOR = - System.getProperty("line.separator"); + System.getProperty("line.separator"); static String runFsck(Configuration conf, int expectedErrCode, - boolean checkErrorCode,String... path) + boolean checkErrorCode, String... path) throws Exception { ByteArrayOutputStream bStream = new ByteArrayOutputStream(); PrintStream out = new PrintStream(bStream, true); @@ -157,60 +170,72 @@ public class TestFsck { assertEquals(expectedErrCode, errCode); } GenericTestUtils.setLogLevel(FSPermissionChecker.LOG, Level.INFO); - FSImage.LOG.info("OUTPUT = " + bStream.toString()); + LOG.info("OUTPUT = " + bStream.toString()); return bStream.toString(); } - /** do fsck */ + private MiniDFSCluster cluster = null; + private Configuration conf = null; + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + } + + @After + public void tearDown() throws Exception { + shutdownCluster(); + } + + private void shutdownCluster() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + } + + /** do fsck. */ @Test public void testFsck() throws Exception { DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsck"). setNumFiles(20).build(); - MiniDFSCluster cluster = null; FileSystem fs = null; - try { - Configuration conf = new HdfsConfiguration(); - final long precision = 1L; - conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision); - conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); - fs = cluster.getFileSystem(); - final String fileName = "/srcdat"; - util.createFiles(fs, fileName); - util.waitReplication(fs, fileName, (short)3); - final Path file = new Path(fileName); - long aTime = fs.getFileStatus(file).getAccessTime(); - Thread.sleep(precision); - setupAuditLogs(); - String outStr = runFsck(conf, 0, true, "/"); - verifyAuditLogs(); - assertEquals(aTime, fs.getFileStatus(file).getAccessTime()); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - if (fs != null) {try{fs.close();} catch(Exception e){}} - cluster.shutdown(); - - // restart the cluster; bring up namenode but not the data nodes - cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(0).format(false).build(); - outStr = runFsck(conf, 1, true, "/"); - // expect the result is corrupt - assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); - System.out.println(outStr); - - // bring up data nodes & cleanup cluster - cluster.startDataNodes(conf, 4, true, null, null); - cluster.waitActive(); - cluster.waitClusterUp(); - fs = cluster.getFileSystem(); - util.cleanup(fs, "/srcdat"); - } finally { - if (fs != null) {try{fs.close();} catch(Exception e){}} - if (cluster != null) { cluster.shutdown(); } - } + final long precision = 1L; + conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, + precision); + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + fs = cluster.getFileSystem(); + final String fileName = "/srcdat"; + util.createFiles(fs, fileName); + util.waitReplication(fs, fileName, (short)3); + final Path file = new Path(fileName); + long aTime = fs.getFileStatus(file).getAccessTime(); + Thread.sleep(precision); + setupAuditLogs(); + String outStr = runFsck(conf, 0, true, "/"); + verifyAuditLogs(); + assertEquals(aTime, fs.getFileStatus(file).getAccessTime()); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + shutdownCluster(); + + // restart the cluster; bring up namenode but not the data nodes + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(0).format(false).build(); + outStr = runFsck(conf, 1, true, "/"); + // expect the result is corrupt + assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); + System.out.println(outStr); + + // bring up data nodes & cleanup cluster + cluster.startDataNodes(conf, 4, true, null, null); + cluster.waitActive(); + cluster.waitClusterUp(); + fs = cluster.getFileSystem(); + util.cleanup(fs, "/srcdat"); } - /** Sets up log4j logger for auditlogs */ + /** Sets up log4j logger for auditlogs. */ private void setupAuditLogs() throws IOException { File file = new File(AUDITLOG_FILE); if (file.exists()) { @@ -241,11 +266,11 @@ public class TestFsck { line = reader.readLine(); assertNotNull(line); assertTrue("Expected getfileinfo event not found in audit log", - getfileinfoPattern.matcher(line).matches()); + GET_FILE_INFO_PATTERN.matcher(line).matches()); } line = reader.readLine(); assertNotNull(line); - assertTrue("Expected fsck event not found in audit log", fsckPattern + assertTrue("Expected fsck event not found in audit log", FSCK_PATTERN .matcher(line).matches()); assertNull("Unexpected event in audit log", reader.readLine()); } finally { @@ -264,175 +289,155 @@ public class TestFsck { public void testFsckNonExistent() throws Exception { DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsck"). setNumFiles(20).build(); - MiniDFSCluster cluster = null; FileSystem fs = null; - try { - Configuration conf = new HdfsConfiguration(); - conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); - fs = cluster.getFileSystem(); - util.createFiles(fs, "/srcdat"); - util.waitReplication(fs, "/srcdat", (short)3); - String outStr = runFsck(conf, 0, true, "/non-existent"); - assertEquals(-1, outStr.indexOf(NamenodeFsck.HEALTHY_STATUS)); - System.out.println(outStr); - util.cleanup(fs, "/srcdat"); - } finally { - if (fs != null) {try{fs.close();} catch(Exception e){}} - if (cluster != null) { cluster.shutdown(); } - } + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + fs = cluster.getFileSystem(); + util.createFiles(fs, "/srcdat"); + util.waitReplication(fs, "/srcdat", (short)3); + String outStr = runFsck(conf, 0, true, "/non-existent"); + assertEquals(-1, outStr.indexOf(NamenodeFsck.HEALTHY_STATUS)); + System.out.println(outStr); + util.cleanup(fs, "/srcdat"); } - /** Test fsck with permission set on inodes */ + /** Test fsck with permission set on inodes. */ @Test public void testFsckPermission() throws Exception { final DFSTestUtil util = new DFSTestUtil.Builder(). setName(getClass().getSimpleName()).setNumFiles(20).build(); - final Configuration conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - MiniDFSCluster cluster = null; - try { - // Create a cluster with the current user, write some files - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); - final MiniDFSCluster c2 = cluster; - final String dir = "/dfsck"; - final Path dirpath = new Path(dir); - final FileSystem fs = c2.getFileSystem(); + // Create a cluster with the current user, write some files + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + final MiniDFSCluster c2 = cluster; + final String dir = "/dfsck"; + final Path dirpath = new Path(dir); + final FileSystem fs = c2.getFileSystem(); - util.createFiles(fs, dir); - util.waitReplication(fs, dir, (short) 3); - fs.setPermission(dirpath, new FsPermission((short) 0700)); + util.createFiles(fs, dir); + util.waitReplication(fs, dir, (short) 3); + fs.setPermission(dirpath, new FsPermission((short) 0700)); - // run DFSck as another user, should fail with permission issue - UserGroupInformation fakeUGI = UserGroupInformation.createUserForTesting( - "ProbablyNotARealUserName", new String[] { "ShangriLa" }); - fakeUGI.doAs(new PrivilegedExceptionAction() { - @Override - public Object run() throws Exception { - System.out.println(runFsck(conf, -1, true, dir)); - return null; - } - }); - - // set permission and try DFSck again as the fake user, should succeed - fs.setPermission(dirpath, new FsPermission((short) 0777)); - fakeUGI.doAs(new PrivilegedExceptionAction() { - @Override - public Object run() throws Exception { - final String outStr = runFsck(conf, 0, true, dir); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - return null; - } - }); + // run DFSck as another user, should fail with permission issue + UserGroupInformation fakeUGI = UserGroupInformation.createUserForTesting( + "ProbablyNotARealUserName", new String[] {"ShangriLa"}); + fakeUGI.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + System.out.println(runFsck(conf, -1, true, dir)); + return null; + } + }); - util.cleanup(fs, dir); - } finally { - if (cluster != null) { cluster.shutdown(); } - } + // set permission and try DFSck again as the fake user, should succeed + fs.setPermission(dirpath, new FsPermission((short) 0777)); + fakeUGI.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final String outStr = runFsck(conf, 0, true, dir); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + return null; + } + }); + + util.cleanup(fs, dir); } @Test public void testFsckMove() throws Exception { - Configuration conf = new HdfsConfiguration(); - final int DFS_BLOCK_SIZE = 1024; - final int NUM_DATANODES = 4; - conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE); + final int dfsBlockSize = 1024; + final int numDatanodes = 4; + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, dfsBlockSize); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); DFSTestUtil util = new DFSTestUtil("TestFsck", 5, 3, - (5 * DFS_BLOCK_SIZE) + (DFS_BLOCK_SIZE - 1), 5 * DFS_BLOCK_SIZE); - MiniDFSCluster cluster = null; + (5 * dfsBlockSize) + (dfsBlockSize - 1), 5 * dfsBlockSize); FileSystem fs = null; - try { - cluster = new MiniDFSCluster.Builder(conf). - numDataNodes(NUM_DATANODES).build(); - String topDir = "/srcdat"; - fs = cluster.getFileSystem(); - cluster.waitActive(); - util.createFiles(fs, topDir); - util.waitReplication(fs, topDir, (short)3); - String outStr = runFsck(conf, 0, true, "/"); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost", - cluster.getNameNodePort()), conf); - String fileNames[] = util.getFileNames(topDir); - CorruptedTestFile ctFiles[] = new CorruptedTestFile[] { + cluster = new MiniDFSCluster.Builder(conf). + numDataNodes(numDatanodes).build(); + String topDir = "/srcdat"; + fs = cluster.getFileSystem(); + cluster.waitActive(); + util.createFiles(fs, topDir); + util.waitReplication(fs, topDir, (short)3); + String outStr = runFsck(conf, 0, true, "/"); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost", + cluster.getNameNodePort()), conf); + String[] fileNames = util.getFileNames(topDir); + CorruptedTestFile[] ctFiles = new CorruptedTestFile[]{ new CorruptedTestFile(fileNames[0], Sets.newHashSet(0), - dfsClient, NUM_DATANODES, DFS_BLOCK_SIZE), + dfsClient, numDatanodes, dfsBlockSize), new CorruptedTestFile(fileNames[1], Sets.newHashSet(2, 3), - dfsClient, NUM_DATANODES, DFS_BLOCK_SIZE), + dfsClient, numDatanodes, dfsBlockSize), new CorruptedTestFile(fileNames[2], Sets.newHashSet(4), - dfsClient, NUM_DATANODES, DFS_BLOCK_SIZE), + dfsClient, numDatanodes, dfsBlockSize), new CorruptedTestFile(fileNames[3], Sets.newHashSet(0, 1, 2, 3), - dfsClient, NUM_DATANODES, DFS_BLOCK_SIZE), + dfsClient, numDatanodes, dfsBlockSize), new CorruptedTestFile(fileNames[4], Sets.newHashSet(1, 2, 3, 4), - dfsClient, NUM_DATANODES, DFS_BLOCK_SIZE) - }; - int totalMissingBlocks = 0; - for (CorruptedTestFile ctFile : ctFiles) { - totalMissingBlocks += ctFile.getTotalMissingBlocks(); - } - for (CorruptedTestFile ctFile : ctFiles) { - ctFile.removeBlocks(cluster); - } - // Wait for fsck to discover all the missing blocks - while (true) { - outStr = runFsck(conf, 1, false, "/"); - String numMissing = null; - String numCorrupt = null; - for (String line : outStr.split(LINE_SEPARATOR)) { - Matcher m = numMissingBlocksPattern.matcher(line); - if (m.matches()) { - numMissing = m.group(1); - } - m = numCorruptBlocksPattern.matcher(line); - if (m.matches()) { - numCorrupt = m.group(1); - } - if (numMissing != null && numCorrupt != null) { - break; - } + dfsClient, numDatanodes, dfsBlockSize) + }; + int totalMissingBlocks = 0; + for (CorruptedTestFile ctFile : ctFiles) { + totalMissingBlocks += ctFile.getTotalMissingBlocks(); + } + for (CorruptedTestFile ctFile : ctFiles) { + ctFile.removeBlocks(cluster); + } + // Wait for fsck to discover all the missing blocks + while (true) { + outStr = runFsck(conf, 1, false, "/"); + String numMissing = null; + String numCorrupt = null; + for (String line : outStr.split(LINE_SEPARATOR)) { + Matcher m = NUM_MISSING_BLOCKS_PATTERN.matcher(line); + if (m.matches()) { + numMissing = m.group(1); } - if (numMissing == null || numCorrupt == null) { - throw new IOException("failed to find number of missing or corrupt" + - " blocks in fsck output."); + m = NUM_CORRUPT_BLOCKS_PATTERN.matcher(line); + if (m.matches()) { + numCorrupt = m.group(1); } - if (numMissing.equals(Integer.toString(totalMissingBlocks))) { - assertTrue(numCorrupt.equals(Integer.toString(0))); - assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); + if (numMissing != null && numCorrupt != null) { break; } - try { - Thread.sleep(100); - } catch (InterruptedException ignore) { - } } - - // Copy the non-corrupt blocks of corruptFileName to lost+found. - outStr = runFsck(conf, 1, false, "/", "-move"); - FSImage.LOG.info("WATERMELON: outStr = " + outStr); - assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); - - // Make sure that we properly copied the block files from the DataNodes - // to lost+found - for (CorruptedTestFile ctFile : ctFiles) { - ctFile.checkSalvagedRemains(); + if (numMissing == null || numCorrupt == null) { + throw new IOException("failed to find number of missing or corrupt" + + " blocks in fsck output."); + } + if (numMissing.equals(Integer.toString(totalMissingBlocks))) { + assertTrue(numCorrupt.equals(Integer.toString(0))); + assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); + break; + } + try { + Thread.sleep(100); + } catch (InterruptedException ignore) { } - - // Fix the filesystem by removing corruptFileName - outStr = runFsck(conf, 1, true, "/", "-delete"); - assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); - - // Check to make sure we have a healthy filesystem - outStr = runFsck(conf, 0, true, "/"); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - util.cleanup(fs, topDir); - } finally { - if (fs != null) {try{fs.close();} catch(Exception e){}} - if (cluster != null) { cluster.shutdown(); } } + + // Copy the non-corrupt blocks of corruptFileName to lost+found. + outStr = runFsck(conf, 1, false, "/", "-move"); + LOG.info("WATERMELON: outStr = " + outStr); + assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); + + // Make sure that we properly copied the block files from the DataNodes + // to lost+found + for (CorruptedTestFile ctFile : ctFiles) { + ctFile.checkSalvagedRemains(); + } + + // Fix the filesystem by removing corruptFileName + outStr = runFsck(conf, 1, true, "/", "-delete"); + assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); + + // Check to make sure we have a healthy filesystem + outStr = runFsck(conf, 0, true, "/"); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + util.cleanup(fs, topDir); } static private class CorruptedTestFile { @@ -443,7 +448,7 @@ public class TestFsck { final private int blockSize; final private byte[] initialContents; - public CorruptedTestFile(String name, Set blocksToCorrupt, + CorruptedTestFile(String name, Set blocksToCorrupt, DFSClient dfsClient, int numDataNodes, int blockSize) throws IOException { this.name = name; @@ -499,7 +504,7 @@ public class TestFsck { new FileOutputStream(blockFile, false); blockFileStream.write("corrupt".getBytes()); blockFileStream.close(); - FSImage.LOG.info("Corrupted block file " + blockFile); + LOG.info("Corrupted block file " + blockFile); } } } @@ -530,7 +535,9 @@ public class TestFsck { if (blockIdx == (numBlocks - 1)) { // The last block might not be full-length len = (int)(in.getFileLength() % blockSize); - if (len == 0) len = blockBuffer.length; + if (len == 0) { + len = blockBuffer.length; + } } IOUtils.readFully(in, blockBuffer, 0, len); int startIdx = blockIdx * blockSize; @@ -549,218 +556,186 @@ public class TestFsck { @Test public void testFsckMoveAndDelete() throws Exception { - final int MAX_MOVE_TRIES = 5; + final int maxMoveTries = 5; DFSTestUtil util = new DFSTestUtil.Builder(). setName("TestFsckMoveAndDelete").setNumFiles(5).build(); - MiniDFSCluster cluster = null; FileSystem fs = null; - try { - Configuration conf = new HdfsConfiguration(); - conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); - String topDir = "/srcdat"; - fs = cluster.getFileSystem(); - cluster.waitActive(); - util.createFiles(fs, topDir); - util.waitReplication(fs, topDir, (short)3); - String outStr = runFsck(conf, 0, true, "/"); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - - // Corrupt a block by deleting it - String[] fileNames = util.getFileNames(topDir); - DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost", - cluster.getNameNodePort()), conf); - String corruptFileName = fileNames[0]; - ExtendedBlock block = dfsClient.getNamenode().getBlockLocations( - corruptFileName, 0, Long.MAX_VALUE).get(0).getBlock(); - for (int i=0; i<4; i++) { - File blockFile = cluster.getBlockFile(i, block); - if(blockFile != null && blockFile.exists()) { - assertTrue(blockFile.delete()); - } - } + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); + conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + String topDir = "/srcdat"; + fs = cluster.getFileSystem(); + cluster.waitActive(); + util.createFiles(fs, topDir); + util.waitReplication(fs, topDir, (short)3); + String outStr = runFsck(conf, 0, true, "/"); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - // We excpect the filesystem to be corrupted - outStr = runFsck(conf, 1, false, "/"); - while (!outStr.contains(NamenodeFsck.CORRUPT_STATUS)) { - try { - Thread.sleep(100); - } catch (InterruptedException ignore) { - } - outStr = runFsck(conf, 1, false, "/"); - } - - // After a fsck -move, the corrupted file should still exist. - for (int i = 0; i < MAX_MOVE_TRIES; i++) { - outStr = runFsck(conf, 1, true, "/", "-move" ); - assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); - String[] newFileNames = util.getFileNames(topDir); - boolean found = false; - for (String f : newFileNames) { - if (f.equals(corruptFileName)) { - found = true; - break; - } - } - assertTrue(found); + // Corrupt a block by deleting it + String[] fileNames = util.getFileNames(topDir); + DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost", + cluster.getNameNodePort()), conf); + String corruptFileName = fileNames[0]; + ExtendedBlock block = dfsClient.getNamenode().getBlockLocations( + corruptFileName, 0, Long.MAX_VALUE).get(0).getBlock(); + for (int i=0; i<4; i++) { + File blockFile = cluster.getBlockFile(i, block); + if(blockFile != null && blockFile.exists()) { + assertTrue(blockFile.delete()); } - - // Fix the filesystem by moving corrupted files to lost+found - outStr = runFsck(conf, 1, true, "/", "-move", "-delete"); - assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); - - // Check to make sure we have healthy filesystem - outStr = runFsck(conf, 0, true, "/"); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - util.cleanup(fs, topDir); - if (fs != null) {try{fs.close();} catch(Exception e){}} - cluster.shutdown(); - } finally { - if (fs != null) {try{fs.close();} catch(Exception e){}} - if (cluster != null) { cluster.shutdown(); } } + + // We excpect the filesystem to be corrupted + outStr = runFsck(conf, 1, false, "/"); + while (!outStr.contains(NamenodeFsck.CORRUPT_STATUS)) { + try { + Thread.sleep(100); + } catch (InterruptedException ignore) { + } + outStr = runFsck(conf, 1, false, "/"); + } + + // After a fsck -move, the corrupted file should still exist. + for (int i = 0; i < maxMoveTries; i++) { + outStr = runFsck(conf, 1, true, "/", "-move"); + assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); + String[] newFileNames = util.getFileNames(topDir); + boolean found = false; + for (String f : newFileNames) { + if (f.equals(corruptFileName)) { + found = true; + break; + } + } + assertTrue(found); + } + + // Fix the filesystem by moving corrupted files to lost+found + outStr = runFsck(conf, 1, true, "/", "-move", "-delete"); + assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); + + // Check to make sure we have healthy filesystem + outStr = runFsck(conf, 0, true, "/"); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + util.cleanup(fs, topDir); } @Test public void testFsckOpenFiles() throws Exception { DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsck"). setNumFiles(4).build(); - MiniDFSCluster cluster = null; FileSystem fs = null; - try { - Configuration conf = new HdfsConfiguration(); - conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); - String topDir = "/srcdat"; - String randomString = "HADOOP "; - fs = cluster.getFileSystem(); - cluster.waitActive(); - util.createFiles(fs, topDir); - util.waitReplication(fs, topDir, (short)3); - String outStr = runFsck(conf, 0, true, "/"); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - // Open a file for writing and do not close for now - Path openFile = new Path(topDir + "/openFile"); - FSDataOutputStream out = fs.create(openFile); - int writeCount = 0; - while (writeCount != 100) { - out.write(randomString.getBytes()); - writeCount++; - } - ((DFSOutputStream) out.getWrappedStream()).hflush(); - // We expect the filesystem to be HEALTHY and show one open file - outStr = runFsck(conf, 0, true, topDir); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - assertFalse(outStr.contains("OPENFORWRITE")); - // Use -openforwrite option to list open files - outStr = runFsck(conf, 0, true, topDir, "-files", "-blocks", - "-locations", "-openforwrite"); - System.out.println(outStr); - assertTrue(outStr.contains("OPENFORWRITE")); - assertTrue(outStr.contains("Under Construction Block:")); - assertTrue(outStr.contains("openFile")); - // Close the file - out.close(); - // Now, fsck should show HEALTHY fs and should not show any open files - outStr = runFsck(conf, 0, true, topDir); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - assertFalse(outStr.contains("OPENFORWRITE")); - assertFalse(outStr.contains("Under Construction Block:")); - util.cleanup(fs, topDir); - if (fs != null) {try{fs.close();} catch(Exception e){}} - cluster.shutdown(); - } finally { - if (fs != null) {try{fs.close();} catch(Exception e){}} - if (cluster != null) { cluster.shutdown(); } + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + String topDir = "/srcdat"; + String randomString = "HADOOP "; + fs = cluster.getFileSystem(); + cluster.waitActive(); + util.createFiles(fs, topDir); + util.waitReplication(fs, topDir, (short)3); + String outStr = runFsck(conf, 0, true, "/"); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + // Open a file for writing and do not close for now + Path openFile = new Path(topDir + "/openFile"); + FSDataOutputStream out = fs.create(openFile); + int writeCount = 0; + while (writeCount != 100) { + out.write(randomString.getBytes()); + writeCount++; } + ((DFSOutputStream) out.getWrappedStream()).hflush(); + // We expect the filesystem to be HEALTHY and show one open file + outStr = runFsck(conf, 0, true, topDir); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + assertFalse(outStr.contains("OPENFORWRITE")); + // Use -openforwrite option to list open files + outStr = runFsck(conf, 0, true, topDir, "-files", "-blocks", + "-locations", "-openforwrite"); + System.out.println(outStr); + assertTrue(outStr.contains("OPENFORWRITE")); + assertTrue(outStr.contains("Under Construction Block:")); + assertTrue(outStr.contains("openFile")); + // Close the file + out.close(); + // Now, fsck should show HEALTHY fs and should not show any open files + outStr = runFsck(conf, 0, true, topDir); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + assertFalse(outStr.contains("OPENFORWRITE")); + assertFalse(outStr.contains("Under Construction Block:")); + util.cleanup(fs, topDir); } @Test public void testFsckOpenECFiles() throws Exception { DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsckECFile"). setNumFiles(4).build(); - Configuration conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy(); int numAllUnits = ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes( + cluster = new MiniDFSCluster.Builder(conf).numDataNodes( numAllUnits + 1).build(); FileSystem fs = null; - try { - String topDir = "/myDir"; - byte[] randomBytes = new byte[3000000]; - int seed = 42; - new Random(seed).nextBytes(randomBytes); - cluster.waitActive(); - fs = cluster.getFileSystem(); - util.createFiles(fs, topDir); - // set topDir to EC when it has replicated files - cluster.getFileSystem().getClient().setErasureCodingPolicy( - topDir, ecPolicy); + String topDir = "/myDir"; + byte[] randomBytes = new byte[3000000]; + int seed = 42; + new Random(seed).nextBytes(randomBytes); + cluster.waitActive(); + fs = cluster.getFileSystem(); + util.createFiles(fs, topDir); + // set topDir to EC when it has replicated files + cluster.getFileSystem().getClient().setErasureCodingPolicy( + topDir, ecPolicy); - // create a new file under topDir - DFSTestUtil.createFile(fs, new Path(topDir, "ecFile"), 1024, (short) 1, 0L); - // Open a EC file for writing and do not close for now - Path openFile = new Path(topDir + "/openECFile"); - FSDataOutputStream out = fs.create(openFile); - int writeCount = 0; - while (writeCount != 300) { - out.write(randomBytes); - writeCount++; - } - - // make sure the fsck can correctly handle mixed ec/replicated files - runFsck(conf, 0, true, topDir, "-files", "-blocks", "-openforwrite"); - - // We expect the filesystem to be HEALTHY and show one open file - String outStr = runFsck(conf, 0, true, openFile.toString(), "-files", - "-blocks", "-openforwrite"); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - assertTrue(outStr.contains("OPENFORWRITE")); - assertTrue(outStr.contains("Live_repl=" + numAllUnits)); - assertTrue(outStr.contains("Expected_repl=" + numAllUnits)); - - // Use -openforwrite option to list open files - outStr = runFsck(conf, 0, true, openFile.toString(), "-files", "-blocks", - "-locations", "-openforwrite", "-replicaDetails"); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - assertTrue(outStr.contains("OPENFORWRITE")); - assertTrue(outStr.contains("Live_repl=" + numAllUnits)); - assertTrue(outStr.contains("Expected_repl=" + numAllUnits)); - assertTrue(outStr.contains("Under Construction Block:")); - - // Close the file - out.close(); - - // Now, fsck should show HEALTHY fs and should not show any open files - outStr = runFsck(conf, 0, true, openFile.toString(), "-files", "-blocks", - "-locations", "-racks", "-replicaDetails"); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - assertFalse(outStr.contains("OPENFORWRITE")); - assertFalse(outStr.contains("Under Construction Block:")); - assertFalse(outStr.contains("Expected_repl=" + numAllUnits)); - assertTrue(outStr.contains("Live_repl=" + numAllUnits)); - util.cleanup(fs, topDir); - } finally { - if (fs != null) { - try { - fs.close(); - } catch (Exception e) { - } - } - if (cluster != null) { - cluster.shutdown(); - } + // create a new file under topDir + DFSTestUtil.createFile(fs, new Path(topDir, "ecFile"), 1024, (short) 1, 0L); + // Open a EC file for writing and do not close for now + Path openFile = new Path(topDir + "/openECFile"); + FSDataOutputStream out = fs.create(openFile); + int writeCount = 0; + while (writeCount != 300) { + out.write(randomBytes); + writeCount++; } + + // make sure the fsck can correctly handle mixed ec/replicated files + runFsck(conf, 0, true, topDir, "-files", "-blocks", "-openforwrite"); + + // We expect the filesystem to be HEALTHY and show one open file + String outStr = runFsck(conf, 0, true, openFile.toString(), "-files", + "-blocks", "-openforwrite"); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + assertTrue(outStr.contains("OPENFORWRITE")); + assertTrue(outStr.contains("Live_repl=" + numAllUnits)); + assertTrue(outStr.contains("Expected_repl=" + numAllUnits)); + + // Use -openforwrite option to list open files + outStr = runFsck(conf, 0, true, openFile.toString(), "-files", "-blocks", + "-locations", "-openforwrite", "-replicaDetails"); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + assertTrue(outStr.contains("OPENFORWRITE")); + assertTrue(outStr.contains("Live_repl=" + numAllUnits)); + assertTrue(outStr.contains("Expected_repl=" + numAllUnits)); + assertTrue(outStr.contains("Under Construction Block:")); + + // Close the file + out.close(); + + // Now, fsck should show HEALTHY fs and should not show any open files + outStr = runFsck(conf, 0, true, openFile.toString(), "-files", "-blocks", + "-locations", "-racks", "-replicaDetails"); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + assertFalse(outStr.contains("OPENFORWRITE")); + assertFalse(outStr.contains("Under Construction Block:")); + assertFalse(outStr.contains("Expected_repl=" + numAllUnits)); + assertTrue(outStr.contains("Live_repl=" + numAllUnits)); + util.cleanup(fs, topDir); } @Test public void testCorruptBlock() throws Exception { - Configuration conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000); // Set short retry timeouts so this test runs faster conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10); @@ -772,8 +747,6 @@ public class TestFsck { String outStr = null; short factor = 1; - MiniDFSCluster cluster = null; - try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); fs = cluster.getFileSystem(); @@ -804,7 +777,7 @@ public class TestFsck { IOUtils.copyBytes(fs.open(file1), new IOUtils.NullOutputStream(), conf, true); } catch (IOException ie) { - // Ignore exception + assertTrue(ie instanceof ChecksumException); } dfsClient = new DFSClient(new InetSocketAddress("localhost", @@ -821,27 +794,23 @@ public class TestFsck { getBlockLocations(file1.toString(), 0, Long.MAX_VALUE); replicaCount = blocks.get(0).getLocations().length; } - assertTrue (blocks.get(0).isCorrupt()); + assertTrue(blocks.get(0).isCorrupt()); // Check if fsck reports the same outStr = runFsck(conf, 1, true, "/"); System.out.println(outStr); assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); assertTrue(outStr.contains("testCorruptBlock")); - } finally { - if (cluster != null) {cluster.shutdown();} - } } @Test public void testUnderMinReplicatedBlock() throws Exception { - Configuration conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000); // Set short retry timeouts so this test runs faster conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10); // Set minReplication to 2 short minReplication=2; - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY,minReplication); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, minReplication); FileSystem fs = null; DFSClient dfsClient = null; LocatedBlocks blocks = null; @@ -849,252 +818,234 @@ public class TestFsck { Random random = new Random(); String outStr = null; short factor = 1; - MiniDFSCluster cluster = null; - try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - cluster.waitActive(); - fs = cluster.getFileSystem(); - Path file1 = new Path("/testUnderMinReplicatedBlock"); - DFSTestUtil.createFile(fs, file1, 1024, minReplication, 0); - // Wait until file replication has completed - DFSTestUtil.waitReplication(fs, file1, minReplication); - ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + Path file1 = new Path("/testUnderMinReplicatedBlock"); + DFSTestUtil.createFile(fs, file1, 1024, minReplication, 0); + // Wait until file replication has completed + DFSTestUtil.waitReplication(fs, file1, minReplication); + ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1); - // Make sure filesystem is in healthy state - outStr = runFsck(conf, 0, true, "/"); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + // Make sure filesystem is in healthy state + outStr = runFsck(conf, 0, true, "/"); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - // corrupt the first replica - File blockFile = cluster.getBlockFile(0, block); - if (blockFile != null && blockFile.exists()) { - RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw"); - FileChannel channel = raFile.getChannel(); - String badString = "BADBAD"; - int rand = random.nextInt((int) channel.size()/2); - raFile.seek(rand); - raFile.write(badString.getBytes()); - raFile.close(); + // corrupt the first replica + File blockFile = cluster.getBlockFile(0, block); + if (blockFile != null && blockFile.exists()) { + RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw"); + FileChannel channel = raFile.getChannel(); + String badString = "BADBAD"; + int rand = random.nextInt((int) channel.size()/2); + raFile.seek(rand); + raFile.write(badString.getBytes()); + raFile.close(); + } + + dfsClient = new DFSClient(new InetSocketAddress("localhost", + cluster.getNameNodePort()), conf); + blocks = dfsClient.getNamenode(). + getBlockLocations(file1.toString(), 0, Long.MAX_VALUE); + replicaCount = blocks.get(0).getLocations().length; + while (replicaCount != factor) { + try { + Thread.sleep(100); + // Read the file to trigger reportBadBlocks + try { + IOUtils.copyBytes(fs.open(file1), new IOUtils.NullOutputStream(), + conf, true); + } catch (IOException ie) { + assertTrue(ie instanceof ChecksumException); + } + System.out.println("sleep in try: replicaCount=" + replicaCount + + " factor=" + factor); + } catch (InterruptedException ignore) { } - - dfsClient = new DFSClient(new InetSocketAddress("localhost", - cluster.getNameNodePort()), conf); blocks = dfsClient.getNamenode(). getBlockLocations(file1.toString(), 0, Long.MAX_VALUE); replicaCount = blocks.get(0).getLocations().length; - while (replicaCount != factor) { - try { - Thread.sleep(100); - // Read the file to trigger reportBadBlocks - try { - IOUtils.copyBytes(fs.open(file1), new IOUtils.NullOutputStream(), conf, - true); - } catch (IOException ie) { - // Ignore exception - } - System.out.println("sleep in try: replicaCount="+replicaCount+" factor="+factor); - } catch (InterruptedException ignore) { - } - blocks = dfsClient.getNamenode(). - getBlockLocations(file1.toString(), 0, Long.MAX_VALUE); - replicaCount = blocks.get(0).getLocations().length; - } - - // Check if fsck reports the same - outStr = runFsck(conf, 0, true, "/"); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - assertTrue(outStr.contains("UNDER MIN REPL'D BLOCKS:\t1 (100.0 %)")); - assertTrue(outStr.contains("dfs.namenode.replication.min:\t2")); - } finally { - if (cluster != null) {cluster.shutdown();} } + + // Check if fsck reports the same + outStr = runFsck(conf, 0, true, "/"); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + assertTrue(outStr.contains("UNDER MIN REPL'D BLOCKS:\t1 (100.0 %)")); + assertTrue(outStr.contains("dfs.namenode.replication.min:\t2")); } @Test(timeout = 60000) public void testFsckReplicaDetails() throws Exception { - final short REPL_FACTOR = 1; - short NUM_DN = 1; + final short replFactor = 1; + short numDn = 1; final long blockSize = 512; final long fileSize = 1024; boolean checkDecommissionInProgress = false; - String[] racks = { "/rack1" }; - String[] hosts = { "host1" }; + String[] racks = {"/rack1"}; + String[] hosts = {"host1"}; - Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); - MiniDFSCluster cluster; DistributedFileSystem dfs; cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts).racks(racks).build(); + new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) + .racks(racks).build(); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); // create files final String testFile = new String("/testfile"); final Path path = new Path(testFile); - DFSTestUtil.createFile(dfs, path, fileSize, REPL_FACTOR, 1000L); - DFSTestUtil.waitReplication(dfs, path, REPL_FACTOR); + DFSTestUtil.createFile(dfs, path, fileSize, replFactor, 1000L); + DFSTestUtil.waitReplication(dfs, path, replFactor); + + // make sure datanode that has replica is fine before decommission + String fsckOut = runFsck(conf, 0, true, testFile, "-files", "-blocks", + "-replicaDetails"); + assertTrue(fsckOut.contains(NamenodeFsck.HEALTHY_STATUS)); + assertTrue(fsckOut.contains("(LIVE)")); + + // decommission datanode + ExtendedBlock eb = DFSTestUtil.getFirstBlock(dfs, path); + FSNamesystem fsn = cluster.getNameNode().getNamesystem(); + BlockManager bm = fsn.getBlockManager(); + BlockCollection bc = null; try { - // make sure datanode that has replica is fine before decommission - String fsckOut = runFsck(conf, 0, true, testFile, "-files", "-blocks", "-replicaDetails"); - assertTrue(fsckOut.contains(NamenodeFsck.HEALTHY_STATUS)); - assertTrue(fsckOut.contains("(LIVE)")); - - // decommission datanode - ExtendedBlock eb = DFSTestUtil.getFirstBlock(dfs, path); - FSNamesystem fsn = cluster.getNameNode().getNamesystem(); - BlockManager bm = fsn.getBlockManager(); - BlockCollection bc = null; - try { - fsn.writeLock(); - BlockInfo bi = bm.getStoredBlock(eb.getLocalBlock()); - bc = fsn.getBlockCollection(bi); - } finally { - fsn.writeUnlock(); - } - DatanodeDescriptor dn = bc.getBlocks()[0] - .getDatanode(0); - bm.getDatanodeManager().getDecomManager().startDecommission(dn); - String dnName = dn.getXferAddr(); - - // check the replica status while decommissioning - fsckOut = runFsck(conf, 0, true, testFile, "-files", "-blocks", "-replicaDetails"); - assertTrue(fsckOut.contains("(DECOMMISSIONING)")); - - // Start 2nd Datanode and wait for decommission to start - cluster.startDataNodes(conf, 1, true, null, null, null); - DatanodeInfo datanodeInfo = null; - do { - Thread.sleep(2000); - for (DatanodeInfo info : dfs.getDataNodeStats()) { - if (dnName.equals(info.getXferAddr())) { - datanodeInfo = info; - } - } - if (!checkDecommissionInProgress && datanodeInfo != null - && datanodeInfo.isDecommissionInProgress()) { - checkDecommissionInProgress = true; - } - } while (datanodeInfo != null && !datanodeInfo.isDecommissioned()); - - // check the replica status after decommission is done - fsckOut = runFsck(conf, 0, true, testFile, "-files", "-blocks", "-replicaDetails"); - assertTrue(fsckOut.contains("(DECOMMISSIONED)")); + fsn.writeLock(); + BlockInfo bi = bm.getStoredBlock(eb.getLocalBlock()); + bc = fsn.getBlockCollection(bi); } finally { - if (cluster != null) { - cluster.shutdown(); - } + fsn.writeUnlock(); } + DatanodeDescriptor dn = bc.getBlocks()[0] + .getDatanode(0); + bm.getDatanodeManager().getDecomManager().startDecommission(dn); + String dnName = dn.getXferAddr(); + + // check the replica status while decommissioning + fsckOut = runFsck(conf, 0, true, testFile, "-files", "-blocks", + "-replicaDetails"); + assertTrue(fsckOut.contains("(DECOMMISSIONING)")); + + // Start 2nd Datanode and wait for decommission to start + cluster.startDataNodes(conf, 1, true, null, null, null); + DatanodeInfo datanodeInfo = null; + do { + Thread.sleep(2000); + for (DatanodeInfo info : dfs.getDataNodeStats()) { + if (dnName.equals(info.getXferAddr())) { + datanodeInfo = info; + } + } + if (!checkDecommissionInProgress && datanodeInfo != null + && datanodeInfo.isDecommissionInProgress()) { + checkDecommissionInProgress = true; + } + } while (datanodeInfo != null && !datanodeInfo.isDecommissioned()); + + // check the replica status after decommission is done + fsckOut = runFsck(conf, 0, true, testFile, "-files", "-blocks", + "-replicaDetails"); + assertTrue(fsckOut.contains("(DECOMMISSIONED)")); } - /** Test if fsck can return -1 in case of failure + /** Test if fsck can return -1 in case of failure. * * @throws Exception */ @Test public void testFsckError() throws Exception { - MiniDFSCluster cluster = null; - try { - // bring up a one-node cluster - Configuration conf = new HdfsConfiguration(); - cluster = new MiniDFSCluster.Builder(conf).build(); - String fileName = "/test.txt"; - Path filePath = new Path(fileName); - FileSystem fs = cluster.getFileSystem(); - - // create a one-block file - DFSTestUtil.createFile(fs, filePath, 1L, (short)1, 1L); - DFSTestUtil.waitReplication(fs, filePath, (short)1); - - // intentionally corrupt NN data structure - INodeFile node = (INodeFile) cluster.getNamesystem().dir.getINode - (fileName, true); - final BlockInfo[] blocks = node.getBlocks(); - assertEquals(blocks.length, 1); - blocks[0].setNumBytes(-1L); // set the block length to be negative - - // run fsck and expect a failure with -1 as the error code - String outStr = runFsck(conf, -1, true, fileName); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.FAILURE_STATUS)); - - // clean up file system - fs.delete(filePath, true); - } finally { - if (cluster != null) {cluster.shutdown();} - } + // bring up a one-node cluster + cluster = new MiniDFSCluster.Builder(conf).build(); + String fileName = "/test.txt"; + Path filePath = new Path(fileName); + FileSystem fs = cluster.getFileSystem(); + + // create a one-block file + DFSTestUtil.createFile(fs, filePath, 1L, (short)1, 1L); + DFSTestUtil.waitReplication(fs, filePath, (short)1); + + // intentionally corrupt NN data structure + INodeFile node = (INodeFile) cluster.getNamesystem().dir.getINode( + fileName, true); + final BlockInfo[] blocks = node.getBlocks(); + assertEquals(blocks.length, 1); + blocks[0].setNumBytes(-1L); // set the block length to be negative + + // run fsck and expect a failure with -1 as the error code + String outStr = runFsck(conf, -1, true, fileName); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.FAILURE_STATUS)); + + // clean up file system + fs.delete(filePath, true); } - /** check if option -list-corruptfiles of fsck command works properly */ + /** check if option -list-corruptfiles of fsck command works properly. */ @Test public void testFsckListCorruptFilesBlocks() throws Exception { - Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000); conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); FileSystem fs = null; - MiniDFSCluster cluster = null; - try { - cluster = new MiniDFSCluster.Builder(conf).build(); - cluster.waitActive(); - fs = cluster.getFileSystem(); - DFSTestUtil util = new DFSTestUtil.Builder(). - setName("testGetCorruptFiles").setNumFiles(3).setMaxLevels(1). - setMaxSize(1024).build(); - util.createFiles(fs, "/corruptData", (short) 1); - util.waitReplication(fs, "/corruptData", (short) 1); + cluster = new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + DFSTestUtil util = new DFSTestUtil.Builder(). + setName("testGetCorruptFiles").setNumFiles(3).setMaxLevels(1). + setMaxSize(1024).build(); + util.createFiles(fs, "/corruptData", (short) 1); + util.waitReplication(fs, "/corruptData", (short) 1); - // String outStr = runFsck(conf, 0, true, "/corruptData", "-list-corruptfileblocks"); - String outStr = runFsck(conf, 0, false, "/corruptData", "-list-corruptfileblocks"); - System.out.println("1. good fsck out: " + outStr); - assertTrue(outStr.contains("has 0 CORRUPT files")); - // delete the blocks - final String bpid = cluster.getNamesystem().getBlockPoolId(); - for (int i=0; i<4; i++) { - for (int j=0; j<=1; j++) { - File storageDir = cluster.getInstanceStorageDir(i, j); - File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); - List metadataFiles = MiniDFSCluster.getAllBlockMetadataFiles( - data_dir); - if (metadataFiles == null) - continue; - for (File metadataFile : metadataFiles) { - File blockFile = Block.metaToBlockFile(metadataFile); - assertTrue("Cannot remove file.", blockFile.delete()); - assertTrue("Cannot remove file.", metadataFile.delete()); - } + String outStr = runFsck(conf, 0, false, "/corruptData", + "-list-corruptfileblocks"); + System.out.println("1. good fsck out: " + outStr); + assertTrue(outStr.contains("has 0 CORRUPT files")); + // delete the blocks + final String bpid = cluster.getNamesystem().getBlockPoolId(); + for (int i=0; i<4; i++) { + for (int j=0; j<=1; j++) { + File storageDir = cluster.getInstanceStorageDir(i, j); + File dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); + List metadataFiles = MiniDFSCluster.getAllBlockMetadataFiles( + dataDir); + if (metadataFiles == null) { + continue; + } + for (File metadataFile : metadataFiles) { + File blockFile = Block.metaToBlockFile(metadataFile); + assertTrue("Cannot remove file.", blockFile.delete()); + assertTrue("Cannot remove file.", metadataFile.delete()); } } - - // wait for the namenode to see the corruption - final NamenodeProtocols namenode = cluster.getNameNodeRpc(); - CorruptFileBlocks corruptFileBlocks = namenode - .listCorruptFileBlocks("/corruptData", null); - int numCorrupt = corruptFileBlocks.getFiles().length; - while (numCorrupt == 0) { - Thread.sleep(1000); - corruptFileBlocks = namenode - .listCorruptFileBlocks("/corruptData", null); - numCorrupt = corruptFileBlocks.getFiles().length; - } - outStr = runFsck(conf, -1, true, "/corruptData", "-list-corruptfileblocks"); - System.out.println("2. bad fsck out: " + outStr); - assertTrue(outStr.contains("has 3 CORRUPT files")); - - // Do a listing on a dir which doesn't have any corrupt blocks and validate - util.createFiles(fs, "/goodData"); - outStr = runFsck(conf, 0, true, "/goodData", "-list-corruptfileblocks"); - System.out.println("3. good fsck out: " + outStr); - assertTrue(outStr.contains("has 0 CORRUPT files")); - util.cleanup(fs,"/corruptData"); - util.cleanup(fs, "/goodData"); - } finally { - if (cluster != null) {cluster.shutdown();} } + + // wait for the namenode to see the corruption + final NamenodeProtocols namenode = cluster.getNameNodeRpc(); + CorruptFileBlocks corruptFileBlocks = namenode + .listCorruptFileBlocks("/corruptData", null); + int numCorrupt = corruptFileBlocks.getFiles().length; + while (numCorrupt == 0) { + Thread.sleep(1000); + corruptFileBlocks = namenode + .listCorruptFileBlocks("/corruptData", null); + numCorrupt = corruptFileBlocks.getFiles().length; + } + outStr = runFsck(conf, -1, true, "/corruptData", "-list-corruptfileblocks"); + System.out.println("2. bad fsck out: " + outStr); + assertTrue(outStr.contains("has 3 CORRUPT files")); + + // Do a listing on a dir which doesn't have any corrupt blocks and validate + util.createFiles(fs, "/goodData"); + outStr = runFsck(conf, 0, true, "/goodData", "-list-corruptfileblocks"); + System.out.println("3. good fsck out: " + outStr); + assertTrue(outStr.contains("has 0 CORRUPT files")); + util.cleanup(fs, "/corruptData"); + util.cleanup(fs, "/goodData"); } /** @@ -1103,193 +1054,163 @@ public class TestFsck { */ @Test public void testToCheckTheFsckCommandOnIllegalArguments() throws Exception { - MiniDFSCluster cluster = null; - try { - // bring up a one-node cluster - Configuration conf = new HdfsConfiguration(); - cluster = new MiniDFSCluster.Builder(conf).build(); - String fileName = "/test.txt"; - Path filePath = new Path(fileName); - FileSystem fs = cluster.getFileSystem(); + // bring up a one-node cluster + cluster = new MiniDFSCluster.Builder(conf).build(); + String fileName = "/test.txt"; + Path filePath = new Path(fileName); + FileSystem fs = cluster.getFileSystem(); - // create a one-block file - DFSTestUtil.createFile(fs, filePath, 1L, (short) 1, 1L); - DFSTestUtil.waitReplication(fs, filePath, (short) 1); + // create a one-block file + DFSTestUtil.createFile(fs, filePath, 1L, (short) 1, 1L); + DFSTestUtil.waitReplication(fs, filePath, (short) 1); - // passing illegal option - String outStr = runFsck(conf, -1, true, fileName, "-thisIsNotAValidFlag"); - System.out.println(outStr); - assertTrue(!outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + // passing illegal option + String outStr = runFsck(conf, -1, true, fileName, "-thisIsNotAValidFlag"); + System.out.println(outStr); + assertTrue(!outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - // passing multiple paths are arguments - outStr = runFsck(conf, -1, true, "/", fileName); - System.out.println(outStr); - assertTrue(!outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - // clean up file system - fs.delete(filePath, true); - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } + // passing multiple paths are arguments + outStr = runFsck(conf, -1, true, "/", fileName); + System.out.println(outStr); + assertTrue(!outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + // clean up file system + fs.delete(filePath, true); } /** - * Tests that the # of missing block replicas and expected replicas is correct + * Tests that the # of missing block replicas and expected replicas is + * correct. * @throws IOException */ @Test public void testFsckMissingReplicas() throws IOException { // Desired replication factor - // Set this higher than NUM_REPLICAS so it's under-replicated - final short REPL_FACTOR = 2; + // Set this higher than numReplicas so it's under-replicated + final short replFactor = 2; // Number of replicas to actually start - final short NUM_REPLICAS = 1; + final short numReplicas = 1; // Number of blocks to write - final short NUM_BLOCKS = 3; + final short numBlocks = 3; // Set a small-ish blocksize final long blockSize = 512; - Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; - try { - // Startup a minicluster - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(NUM_REPLICAS).build(); - assertNotNull("Failed Cluster Creation", cluster); - cluster.waitClusterUp(); - dfs = cluster.getFileSystem(); - assertNotNull("Failed to get FileSystem", dfs); - - // Create a file that will be intentionally under-replicated - final String pathString = new String("/testfile"); - final Path path = new Path(pathString); - long fileLen = blockSize * NUM_BLOCKS; - DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); - - // Create an under-replicated file - NameNode namenode = cluster.getNameNode(); - NetworkTopology nettop = cluster.getNamesystem().getBlockManager() - .getDatanodeManager().getNetworkTopology(); - Map pmap = new HashMap(); - Writer result = new StringWriter(); - PrintWriter out = new PrintWriter(result, true); - InetAddress remoteAddress = InetAddress.getLocalHost(); - NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, - NUM_REPLICAS, remoteAddress); - - // Run the fsck and check the Result - final HdfsFileStatus file = - namenode.getRpcServer().getFileInfo(pathString); - assertNotNull(file); - Result replRes = new ReplicationResult(conf); - Result ecRes = new ErasureCodingResult(conf); - fsck.check(pathString, file, replRes, ecRes); - // Also print the output from the fsck, for ex post facto sanity checks - System.out.println(result.toString()); - assertEquals(replRes.missingReplicas, - (NUM_BLOCKS*REPL_FACTOR) - (NUM_BLOCKS*NUM_REPLICAS)); - assertEquals(replRes.numExpectedReplicas, NUM_BLOCKS*REPL_FACTOR); - } finally { - if(dfs != null) { - dfs.close(); - } - if(cluster != null) { - cluster.shutdown(); - } - } + // Startup a minicluster + cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(numReplicas).build(); + assertNotNull("Failed Cluster Creation", cluster); + cluster.waitClusterUp(); + dfs = cluster.getFileSystem(); + assertNotNull("Failed to get FileSystem", dfs); + + // Create a file that will be intentionally under-replicated + final String pathString = new String("/testfile"); + final Path path = new Path(pathString); + long fileLen = blockSize * numBlocks; + DFSTestUtil.createFile(dfs, path, fileLen, replFactor, 1); + + // Create an under-replicated file + NameNode namenode = cluster.getNameNode(); + NetworkTopology nettop = cluster.getNamesystem().getBlockManager() + .getDatanodeManager().getNetworkTopology(); + Map pmap = new HashMap(); + Writer result = new StringWriter(); + PrintWriter out = new PrintWriter(result, true); + InetAddress remoteAddress = InetAddress.getLocalHost(); + NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, + numReplicas, remoteAddress); + + // Run the fsck and check the Result + final HdfsFileStatus file = + namenode.getRpcServer().getFileInfo(pathString); + assertNotNull(file); + Result replRes = new ReplicationResult(conf); + Result ecRes = new ErasureCodingResult(conf); + fsck.check(pathString, file, replRes, ecRes); + // Also print the output from the fsck, for ex post facto sanity checks + System.out.println(result.toString()); + assertEquals(replRes.missingReplicas, + (numBlocks*replFactor) - (numBlocks*numReplicas)); + assertEquals(replRes.numExpectedReplicas, numBlocks*replFactor); } /** - * Tests that the # of misreplaced replicas is correct + * Tests that the # of misreplaced replicas is correct. * @throws IOException */ @Test public void testFsckMisPlacedReplicas() throws IOException { // Desired replication factor - final short REPL_FACTOR = 2; + final short replFactor = 2; // Number of replicas to actually start - short NUM_DN = 2; + short numDn = 2; // Number of blocks to write - final short NUM_BLOCKS = 3; + final short numBlocks = 3; // Set a small-ish blocksize final long blockSize = 512; - String [] racks = {"/rack1", "/rack1"}; - String [] hosts = {"host1", "host2"}; + String[] racks = {"/rack1", "/rack1"}; + String[] hosts = {"host1", "host2"}; - Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; - try { - // Startup a minicluster - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts) - .racks(racks).build(); - assertNotNull("Failed Cluster Creation", cluster); - cluster.waitClusterUp(); - dfs = cluster.getFileSystem(); - assertNotNull("Failed to get FileSystem", dfs); - - // Create a file that will be intentionally under-replicated - final String pathString = new String("/testfile"); - final Path path = new Path(pathString); - long fileLen = blockSize * NUM_BLOCKS; - DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); - - // Create an under-replicated file - NameNode namenode = cluster.getNameNode(); - NetworkTopology nettop = cluster.getNamesystem().getBlockManager() - .getDatanodeManager().getNetworkTopology(); - // Add a new node on different rack, so previous blocks' replicas - // are considered to be misplaced - nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2", "/host3")); - NUM_DN++; - - Map pmap = new HashMap(); - Writer result = new StringWriter(); - PrintWriter out = new PrintWriter(result, true); - InetAddress remoteAddress = InetAddress.getLocalHost(); - NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, - NUM_DN, remoteAddress); - - // Run the fsck and check the Result - final HdfsFileStatus file = - namenode.getRpcServer().getFileInfo(pathString); - assertNotNull(file); - Result replRes = new ReplicationResult(conf); - Result ecRes = new ErasureCodingResult(conf); - fsck.check(pathString, file, replRes, ecRes); - // check misReplicatedBlock number. - assertEquals(replRes.numMisReplicatedBlocks, NUM_BLOCKS); - } finally { - if(dfs != null) { - dfs.close(); - } - if(cluster != null) { - cluster.shutdown(); - } - } + // Startup a minicluster + cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) + .racks(racks).build(); + assertNotNull("Failed Cluster Creation", cluster); + cluster.waitClusterUp(); + dfs = cluster.getFileSystem(); + assertNotNull("Failed to get FileSystem", dfs); + + // Create a file that will be intentionally under-replicated + final String pathString = new String("/testfile"); + final Path path = new Path(pathString); + long fileLen = blockSize * numBlocks; + DFSTestUtil.createFile(dfs, path, fileLen, replFactor, 1); + + // Create an under-replicated file + NameNode namenode = cluster.getNameNode(); + NetworkTopology nettop = cluster.getNamesystem().getBlockManager() + .getDatanodeManager().getNetworkTopology(); + // Add a new node on different rack, so previous blocks' replicas + // are considered to be misplaced + nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2", "/host3")); + numDn++; + + Map pmap = new HashMap(); + Writer result = new StringWriter(); + PrintWriter out = new PrintWriter(result, true); + InetAddress remoteAddress = InetAddress.getLocalHost(); + NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, + numDn, remoteAddress); + + // Run the fsck and check the Result + final HdfsFileStatus file = + namenode.getRpcServer().getFileInfo(pathString); + assertNotNull(file); + Result replRes = new ReplicationResult(conf); + Result ecRes = new ErasureCodingResult(conf); + fsck.check(pathString, file, replRes, ecRes); + // check misReplicatedBlock number. + assertEquals(replRes.numMisReplicatedBlocks, numBlocks); } - /** Test fsck with FileNotFound */ + /** Test fsck with FileNotFound. */ @Test public void testFsckFileNotFound() throws Exception { // Number of replicas to actually start - final short NUM_REPLICAS = 1; + final short numReplicas = 1; - Configuration conf = new Configuration(); NameNode namenode = mock(NameNode.class); NetworkTopology nettop = mock(NetworkTopology.class); - Map pmap = new HashMap<>(); + Map pmap = new HashMap<>(); Writer result = new StringWriter(); PrintWriter out = new PrintWriter(result, true); InetAddress remoteAddress = InetAddress.getLocalHost(); @@ -1307,7 +1228,7 @@ public class TestFsck { when(blockManager.getDatanodeManager()).thenReturn(dnManager); NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, - NUM_REPLICAS, remoteAddress); + numReplicas, remoteAddress); String pathString = "/tmp/testFile"; @@ -1320,8 +1241,8 @@ public class TestFsck { FsPermission perms = FsPermission.getDefault(); String owner = "foo"; String group = "bar"; - byte [] symlink = null; - byte [] path = DFSUtil.string2Bytes(pathString); + byte[] symlink = null; + byte[] path = DFSUtil.string2Bytes(pathString); long fileId = 312321L; int numChildren = 1; byte storagePolicy = 0; @@ -1340,95 +1261,82 @@ public class TestFsck { assertTrue(replRes.isHealthy()); } - /** Test fsck with symlinks in the filesystem */ + /** Test fsck with symlinks in the filesystem. */ @Test public void testFsckSymlink() throws Exception { final DFSTestUtil util = new DFSTestUtil.Builder(). setName(getClass().getSimpleName()).setNumFiles(1).build(); - final Configuration conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - MiniDFSCluster cluster = null; FileSystem fs = null; - try { - final long precision = 1L; - conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision); - conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); - fs = cluster.getFileSystem(); - final String fileName = "/srcdat"; - util.createFiles(fs, fileName); - final FileContext fc = FileContext.getFileContext( - cluster.getConfiguration(0)); - final Path file = new Path(fileName); - final Path symlink = new Path("/srcdat-symlink"); - fc.createSymlink(file, symlink, false); - util.waitReplication(fs, fileName, (short)3); - long aTime = fc.getFileStatus(symlink).getAccessTime(); - Thread.sleep(precision); - setupAuditLogs(); - String outStr = runFsck(conf, 0, true, "/"); - verifyAuditLogs(); - assertEquals(aTime, fc.getFileStatus(symlink).getAccessTime()); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - assertTrue(outStr.contains("Total symlinks:\t\t1")); - util.cleanup(fs, fileName); - } finally { - if (fs != null) {try{fs.close();} catch(Exception e){}} - if (cluster != null) { cluster.shutdown(); } - } + final long precision = 1L; + conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, + precision); + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + fs = cluster.getFileSystem(); + final String fileName = "/srcdat"; + util.createFiles(fs, fileName); + final FileContext fc = FileContext.getFileContext( + cluster.getConfiguration(0)); + final Path file = new Path(fileName); + final Path symlink = new Path("/srcdat-symlink"); + fc.createSymlink(file, symlink, false); + util.waitReplication(fs, fileName, (short)3); + long aTime = fc.getFileStatus(symlink).getAccessTime(); + Thread.sleep(precision); + setupAuditLogs(); + String outStr = runFsck(conf, 0, true, "/"); + verifyAuditLogs(); + assertEquals(aTime, fc.getFileStatus(symlink).getAccessTime()); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + assertTrue(outStr.contains("Total symlinks:\t\t1")); + util.cleanup(fs, fileName); } /** - * Test for including the snapshot files in fsck report + * Test for including the snapshot files in fsck report. */ @Test public void testFsckForSnapshotFiles() throws Exception { - final Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) .build(); - try { - String runFsck = runFsck(conf, 0, true, "/", "-includeSnapshots", - "-files"); - assertTrue(runFsck.contains("HEALTHY")); - final String fileName = "/srcdat"; - DistributedFileSystem hdfs = cluster.getFileSystem(); - Path file1 = new Path(fileName); - DFSTestUtil.createFile(hdfs, file1, 1024, (short) 1, 1000L); - hdfs.allowSnapshot(new Path("/")); - hdfs.createSnapshot(new Path("/"), "mySnapShot"); - runFsck = runFsck(conf, 0, true, "/", "-includeSnapshots", "-files"); - assertTrue(runFsck.contains("/.snapshot/mySnapShot/srcdat")); - runFsck = runFsck(conf, 0, true, "/", "-files"); - assertFalse(runFsck.contains("mySnapShot")); - } finally { - cluster.shutdown(); - } + String runFsck = runFsck(conf, 0, true, "/", "-includeSnapshots", + "-files"); + assertTrue(runFsck.contains("HEALTHY")); + final String fileName = "/srcdat"; + DistributedFileSystem hdfs = cluster.getFileSystem(); + Path file1 = new Path(fileName); + DFSTestUtil.createFile(hdfs, file1, 1024, (short) 1, 1000L); + hdfs.allowSnapshot(new Path("/")); + hdfs.createSnapshot(new Path("/"), "mySnapShot"); + runFsck = runFsck(conf, 0, true, "/", "-includeSnapshots", "-files"); + assertTrue(runFsck.contains("/.snapshot/mySnapShot/srcdat")); + runFsck = runFsck(conf, 0, true, "/", "-files"); + assertFalse(runFsck.contains("mySnapShot")); } /** - * Test for blockIdCK + * Test for blockIdCK. */ @Test public void testBlockIdCK() throws Exception { - final short REPL_FACTOR = 2; - short NUM_DN = 2; + final short replFactor = 2; + short numDn = 2; final long blockSize = 512; - String [] racks = {"/rack1", "/rack2"}; - String [] hosts = {"host1", "host2"}; + String[] racks = {"/rack1", "/rack2"}; + String[] hosts = {"host1", "host2"}; - Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2); - MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts) + new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) .racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); @@ -1437,12 +1345,12 @@ public class TestFsck { assertNotNull("Failed to get FileSystem", dfs); DFSTestUtil util = new DFSTestUtil.Builder(). - setName(getClass().getSimpleName()).setNumFiles(1).build(); + setName(getClass().getSimpleName()).setNumFiles(1).build(); //create files final String pathString = new String("/testfile"); final Path path = new Path(pathString); - util.createFile(dfs, path, 1024, REPL_FACTOR , 1000L); - util.waitReplication(dfs, path, REPL_FACTOR); + util.createFile(dfs, path, 1024, replFactor, 1000L); + util.waitReplication(dfs, path, replFactor); StringBuilder sb = new StringBuilder(); for (LocatedBlock lb: util.getAllBlocks(dfs, path)){ sb.append(lb.getBlock().getLocalBlock().getBlockName()+" "); @@ -1450,46 +1358,40 @@ public class TestFsck { String[] bIds = sb.toString().split(" "); //run fsck - try { - //illegal input test - String runFsckResult = runFsck(conf, 0, true, "/", "-blockId", - "not_a_block_id"); - assertTrue(runFsckResult.contains("Incorrect blockId format:")); + //illegal input test + String runFsckResult = runFsck(conf, 0, true, "/", "-blockId", + "not_a_block_id"); + assertTrue(runFsckResult.contains("Incorrect blockId format:")); - //general test - runFsckResult = runFsck(conf, 0, true, "/", "-blockId", sb.toString()); - assertTrue(runFsckResult.contains(bIds[0])); - assertTrue(runFsckResult.contains(bIds[1])); - assertTrue(runFsckResult.contains( - "Block replica on datanode/rack: host1/rack1 is HEALTHY")); - assertTrue(runFsckResult.contains( - "Block replica on datanode/rack: host2/rack2 is HEALTHY")); - } finally { - cluster.shutdown(); - } + //general test + runFsckResult = runFsck(conf, 0, true, "/", "-blockId", sb.toString()); + assertTrue(runFsckResult.contains(bIds[0])); + assertTrue(runFsckResult.contains(bIds[1])); + assertTrue(runFsckResult.contains( + "Block replica on datanode/rack: host1/rack1 is HEALTHY")); + assertTrue(runFsckResult.contains( + "Block replica on datanode/rack: host2/rack2 is HEALTHY")); } /** - * Test for blockIdCK with datanode decommission + * Test for blockIdCK with datanode decommission. */ @Test public void testBlockIdCKDecommission() throws Exception { - final short REPL_FACTOR = 1; - short NUM_DN = 2; + final short replFactor = 1; + short numDn = 2; final long blockSize = 512; boolean checkDecommissionInProgress = false; - String [] racks = {"/rack1", "/rack2"}; - String [] hosts = {"host1", "host2"}; + String[] racks = {"/rack1", "/rack2"}; + String[] hosts = {"host1", "host2"}; - Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2); - MiniDFSCluster cluster; - DistributedFileSystem dfs ; + DistributedFileSystem dfs; cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts) + new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) .racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); @@ -1502,137 +1404,124 @@ public class TestFsck { //create files final String pathString = new String("/testfile"); final Path path = new Path(pathString); - util.createFile(dfs, path, 1024, REPL_FACTOR, 1000L); - util.waitReplication(dfs, path, REPL_FACTOR); + util.createFile(dfs, path, 1024, replFactor, 1000L); + util.waitReplication(dfs, path, replFactor); StringBuilder sb = new StringBuilder(); for (LocatedBlock lb: util.getAllBlocks(dfs, path)){ sb.append(lb.getBlock().getLocalBlock().getBlockName()+" "); } String[] bIds = sb.toString().split(" "); + + //make sure datanode that has replica is fine before decommission + String outStr = runFsck(conf, 0, true, "/", "-blockId", bIds[0]); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + + //decommission datanode + FSNamesystem fsn = cluster.getNameNode().getNamesystem(); + BlockManager bm = fsn.getBlockManager(); + ExtendedBlock eb = util.getFirstBlock(dfs, path); + BlockCollection bc = null; try { - //make sure datanode that has replica is fine before decommission - String outStr = runFsck(conf, 0, true, "/", "-blockId", bIds[0]); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - - //decommission datanode - FSNamesystem fsn = cluster.getNameNode().getNamesystem(); - BlockManager bm = fsn.getBlockManager(); - ExtendedBlock eb = util.getFirstBlock(dfs, path); - BlockCollection bc = null; - try { - fsn.writeLock(); - BlockInfo bi = bm.getStoredBlock(eb.getLocalBlock()); - bc = fsn.getBlockCollection(bi); - } finally { - fsn.writeUnlock(); - } - DatanodeDescriptor dn = bc.getBlocks()[0].getDatanode(0); - bm.getDatanodeManager().getDecomManager().startDecommission(dn); - String dnName = dn.getXferAddr(); - - //wait for decommission start - DatanodeInfo datanodeInfo = null; - int count = 0; - do { - Thread.sleep(2000); - for (DatanodeInfo info : dfs.getDataNodeStats()) { - if (dnName.equals(info.getXferAddr())) { - datanodeInfo = info; - } - } - //check decommissioning only once - if(!checkDecommissionInProgress && datanodeInfo != null - && datanodeInfo.isDecommissionInProgress()) { - String fsckOut = runFsck(conf, 3, true, "/", "-blockId", bIds[0]); - assertTrue(fsckOut.contains(NamenodeFsck.DECOMMISSIONING_STATUS)); - checkDecommissionInProgress = true; - } - } while (datanodeInfo != null && !datanodeInfo.isDecommissioned()); - - //check decommissioned - String fsckOut = runFsck(conf, 2, true, "/", "-blockId", bIds[0]); - assertTrue(fsckOut.contains(NamenodeFsck.DECOMMISSIONED_STATUS)); + fsn.writeLock(); + BlockInfo bi = bm.getStoredBlock(eb.getLocalBlock()); + bc = fsn.getBlockCollection(bi); } finally { - if (cluster != null) { - cluster.shutdown(); - } + fsn.writeUnlock(); } + DatanodeDescriptor dn = bc.getBlocks()[0].getDatanode(0); + bm.getDatanodeManager().getDecomManager().startDecommission(dn); + String dnName = dn.getXferAddr(); + + //wait for decommission start + DatanodeInfo datanodeInfo = null; + int count = 0; + do { + Thread.sleep(2000); + for (DatanodeInfo info : dfs.getDataNodeStats()) { + if (dnName.equals(info.getXferAddr())) { + datanodeInfo = info; + } + } + //check decommissioning only once + if(!checkDecommissionInProgress && datanodeInfo != null + && datanodeInfo.isDecommissionInProgress()) { + String fsckOut = runFsck(conf, 3, true, "/", "-blockId", bIds[0]); + assertTrue(fsckOut.contains(NamenodeFsck.DECOMMISSIONING_STATUS)); + checkDecommissionInProgress = true; + } + } while (datanodeInfo != null && !datanodeInfo.isDecommissioned()); + + //check decommissioned + String fsckOut = runFsck(conf, 2, true, "/", "-blockId", bIds[0]); + assertTrue(fsckOut.contains(NamenodeFsck.DECOMMISSIONED_STATUS)); } /** - * Test for blockIdCK with block corruption + * Test for blockIdCK with block corruption. */ @Test public void testBlockIdCKCorruption() throws Exception { - short NUM_DN = 1; + short numDn = 1; final long blockSize = 512; Random random = new Random(); ExtendedBlock block; short repFactor = 1; - String [] racks = {"/rack1"}; - String [] hosts = {"host1"}; + String[] racks = {"/rack1"}; + String[] hosts = {"host1"}; - Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000); // Set short retry timeouts so this test runs faster conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); - MiniDFSCluster cluster = null; DistributedFileSystem dfs = null; - try { - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts) - .racks(racks).build(); + cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) + .racks(racks).build(); - assertNotNull("Failed Cluster Creation", cluster); - cluster.waitClusterUp(); - dfs = cluster.getFileSystem(); - assertNotNull("Failed to get FileSystem", dfs); + assertNotNull("Failed Cluster Creation", cluster); + cluster.waitClusterUp(); + dfs = cluster.getFileSystem(); + assertNotNull("Failed to get FileSystem", dfs); - DFSTestUtil util = new DFSTestUtil.Builder(). + DFSTestUtil util = new DFSTestUtil.Builder(). setName(getClass().getSimpleName()).setNumFiles(1).build(); - //create files - final String pathString = new String("/testfile"); - final Path path = new Path(pathString); - util.createFile(dfs, path, 1024, repFactor, 1000L); - util.waitReplication(dfs, path, repFactor); - StringBuilder sb = new StringBuilder(); - for (LocatedBlock lb: util.getAllBlocks(dfs, path)){ - sb.append(lb.getBlock().getLocalBlock().getBlockName()+" "); - } - String[] bIds = sb.toString().split(" "); - - //make sure block is healthy before we corrupt it - String outStr = runFsck(conf, 0, true, "/", "-blockId", bIds[0]); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - - // corrupt replicas - block = DFSTestUtil.getFirstBlock(dfs, path); - File blockFile = cluster.getBlockFile(0, block); - if (blockFile != null && blockFile.exists()) { - RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw"); - FileChannel channel = raFile.getChannel(); - String badString = "BADBAD"; - int rand = random.nextInt((int) channel.size()/2); - raFile.seek(rand); - raFile.write(badString.getBytes()); - raFile.close(); - } - - util.waitCorruptReplicas(dfs, cluster.getNamesystem(), path, block, 1); - - outStr = runFsck(conf, 1, false, "/", "-blockId", block.getBlockName()); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); - } finally { - if (cluster != null) { - cluster.shutdown(); - } + //create files + final String pathString = new String("/testfile"); + final Path path = new Path(pathString); + util.createFile(dfs, path, 1024, repFactor, 1000L); + util.waitReplication(dfs, path, repFactor); + StringBuilder sb = new StringBuilder(); + for (LocatedBlock lb: util.getAllBlocks(dfs, path)){ + sb.append(lb.getBlock().getLocalBlock().getBlockName()+" "); } + String[] bIds = sb.toString().split(" "); + + //make sure block is healthy before we corrupt it + String outStr = runFsck(conf, 0, true, "/", "-blockId", bIds[0]); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + + // corrupt replicas + block = DFSTestUtil.getFirstBlock(dfs, path); + File blockFile = cluster.getBlockFile(0, block); + if (blockFile != null && blockFile.exists()) { + RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw"); + FileChannel channel = raFile.getChannel(); + String badString = "BADBAD"; + int rand = random.nextInt((int) channel.size()/2); + raFile.seek(rand); + raFile.write(badString.getBytes()); + raFile.close(); + } + + util.waitCorruptReplicas(dfs, cluster.getNamesystem(), path, block, 1); + + outStr = runFsck(conf, 1, false, "/", "-blockId", block.getBlockName()); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); } private void writeFile(final DistributedFileSystem dfs, @@ -1644,71 +1533,64 @@ public class TestFsck { } private void writeFile(final DistributedFileSystem dfs, - String dirName, String fileName, String StoragePolicy) throws IOException { + String dirName, String fileName, String storagePolicy) + throws IOException { Path dirPath = new Path(dirName); dfs.mkdirs(dirPath); - dfs.setStoragePolicy(dirPath, StoragePolicy); + dfs.setStoragePolicy(dirPath, storagePolicy); writeFile(dfs, dirPath, fileName); } /** - * Test storage policy display + * Test storage policy display. */ @Test public void testStoragePoliciesCK() throws Exception { - final Configuration conf = new HdfsConfiguration(); - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(3) .storageTypes( new StorageType[] {StorageType.DISK, StorageType.ARCHIVE}) .build(); - try { - cluster.waitActive(); - final DistributedFileSystem dfs = cluster.getFileSystem(); - writeFile(dfs, "/testhot", "file", "HOT"); - writeFile(dfs, "/testwarm", "file", "WARM"); - writeFile(dfs, "/testcold", "file", "COLD"); - String outStr = runFsck(conf, 0, true, "/", "-storagepolicies"); - assertTrue(outStr.contains("DISK:3(HOT)")); - assertTrue(outStr.contains("DISK:1,ARCHIVE:2(WARM)")); - assertTrue(outStr.contains("ARCHIVE:3(COLD)")); - assertTrue(outStr.contains("All blocks satisfy specified storage policy.")); - dfs.setStoragePolicy(new Path("/testhot"), "COLD"); - dfs.setStoragePolicy(new Path("/testwarm"), "COLD"); - outStr = runFsck(conf, 0, true, "/", "-storagepolicies"); - assertTrue(outStr.contains("DISK:3(HOT)")); - assertTrue(outStr.contains("DISK:1,ARCHIVE:2(WARM)")); - assertTrue(outStr.contains("ARCHIVE:3(COLD)")); - assertFalse(outStr.contains("All blocks satisfy specified storage policy.")); - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + writeFile(dfs, "/testhot", "file", "HOT"); + writeFile(dfs, "/testwarm", "file", "WARM"); + writeFile(dfs, "/testcold", "file", "COLD"); + String outStr = runFsck(conf, 0, true, "/", "-storagepolicies"); + assertTrue(outStr.contains("DISK:3(HOT)")); + assertTrue(outStr.contains("DISK:1,ARCHIVE:2(WARM)")); + assertTrue(outStr.contains("ARCHIVE:3(COLD)")); + assertTrue(outStr.contains("All blocks satisfy specified storage policy.")); + dfs.setStoragePolicy(new Path("/testhot"), "COLD"); + dfs.setStoragePolicy(new Path("/testwarm"), "COLD"); + outStr = runFsck(conf, 0, true, "/", "-storagepolicies"); + assertTrue(outStr.contains("DISK:3(HOT)")); + assertTrue(outStr.contains("DISK:1,ARCHIVE:2(WARM)")); + assertTrue(outStr.contains("ARCHIVE:3(COLD)")); + assertFalse(outStr.contains( + "All blocks satisfy specified storage policy.")); } /** - * Test for blocks on decommissioning hosts are not shown as missing + * Test for blocks on decommissioning hosts are not shown as missing. */ @Test public void testFsckWithDecommissionedReplicas() throws Exception { - final short REPL_FACTOR = 1; - short NUM_DN = 2; + final short replFactor = 1; + short numDn = 2; final long blockSize = 512; final long fileSize = 1024; boolean checkDecommissionInProgress = false; - String [] racks = {"/rack1", "/rack2"}; - String [] hosts = {"host1", "host2"}; + String[] racks = {"/rack1", "/rack2"}; + String[] hosts = {"host1", "host2"}; - Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); - MiniDFSCluster cluster; - DistributedFileSystem dfs ; + DistributedFileSystem dfs; cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts) + new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) .racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); @@ -1722,114 +1604,106 @@ public class TestFsck { //create files final String testFile = new String("/testfile"); final Path path = new Path(testFile); - util.createFile(dfs, path, fileSize, REPL_FACTOR, 1000L); - util.waitReplication(dfs, path, REPL_FACTOR); + util.createFile(dfs, path, fileSize, replFactor, 1000L); + util.waitReplication(dfs, path, replFactor); + + // make sure datanode that has replica is fine before decommission + String outStr = runFsck(conf, 0, true, testFile); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + + // decommission datanode + FSNamesystem fsn = cluster.getNameNode().getNamesystem(); + BlockManager bm = fsn.getBlockManager(); + ExtendedBlock eb = util.getFirstBlock(dfs, path); + BlockCollection bc = null; try { - // make sure datanode that has replica is fine before decommission - String outStr = runFsck(conf, 0, true, testFile); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - - // decommission datanode - FSNamesystem fsn = cluster.getNameNode().getNamesystem(); - BlockManager bm = fsn.getBlockManager(); - ExtendedBlock eb = util.getFirstBlock(dfs, path); - BlockCollection bc = null; - try { - fsn.writeLock(); - BlockInfo bi = bm.getStoredBlock(eb.getLocalBlock()); - bc = fsn.getBlockCollection(bi); - } finally { - fsn.writeUnlock(); - } - DatanodeDescriptor dn = bc.getBlocks()[0] - .getDatanode(0); - bm.getDatanodeManager().getDecomManager().startDecommission(dn); - String dnName = dn.getXferAddr(); - - // wait for decommission start - DatanodeInfo datanodeInfo = null; - int count = 0; - do { - Thread.sleep(2000); - for (DatanodeInfo info : dfs.getDataNodeStats()) { - if (dnName.equals(info.getXferAddr())) { - datanodeInfo = info; - } - } - // check the replica status should be healthy(0) - // instead of corruption (1) during decommissioning - if(!checkDecommissionInProgress && datanodeInfo != null - && datanodeInfo.isDecommissionInProgress()) { - String fsckOut = runFsck(conf, 0, true, testFile); - checkDecommissionInProgress = true; - } - } while (datanodeInfo != null && !datanodeInfo.isDecommissioned()); - - // check the replica status should be healthy(0) after decommission - // is done - String fsckOut = runFsck(conf, 0, true, testFile); + fsn.writeLock(); + BlockInfo bi = bm.getStoredBlock(eb.getLocalBlock()); + bc = fsn.getBlockCollection(bi); } finally { - if (cluster != null) { - cluster.shutdown(); - } + fsn.writeUnlock(); } + DatanodeDescriptor dn = bc.getBlocks()[0] + .getDatanode(0); + bm.getDatanodeManager().getDecomManager().startDecommission(dn); + String dnName = dn.getXferAddr(); + + // wait for decommission start + DatanodeInfo datanodeInfo = null; + int count = 0; + do { + Thread.sleep(2000); + for (DatanodeInfo info : dfs.getDataNodeStats()) { + if (dnName.equals(info.getXferAddr())) { + datanodeInfo = info; + } + } + // check the replica status should be healthy(0) + // instead of corruption (1) during decommissioning + if(!checkDecommissionInProgress && datanodeInfo != null + && datanodeInfo.isDecommissionInProgress()) { + String fsckOut = runFsck(conf, 0, true, testFile); + checkDecommissionInProgress = true; + } + } while (datanodeInfo != null && !datanodeInfo.isDecommissioned()); + + // check the replica status should be healthy(0) after decommission + // is done + String fsckOut = runFsck(conf, 0, true, testFile); } @Test public void testECFsck() throws Exception { - MiniDFSCluster cluster = null; FileSystem fs = null; - try { - Configuration conf = new HdfsConfiguration(); - final long precision = 1L; - conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision); - conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - int totalSize = ErasureCodingPolicyManager.getSystemDefaultPolicy().getNumDataUnits() - + ErasureCodingPolicyManager.getSystemDefaultPolicy().getNumParityUnits(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(totalSize).build(); - fs = cluster.getFileSystem(); + final long precision = 1L; + conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, + precision); + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); + int dataBlocks = ErasureCodingPolicyManager + .getSystemDefaultPolicy().getNumDataUnits(); + int parityBlocks = ErasureCodingPolicyManager + .getSystemDefaultPolicy().getNumParityUnits(); + int totalSize = dataBlocks + parityBlocks; + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(totalSize).build(); + fs = cluster.getFileSystem(); - // create a contiguous file - Path replDirPath = new Path("/replicated"); - Path replFilePath = new Path(replDirPath, "replfile"); - final short factor = 3; - DFSTestUtil.createFile(fs, replFilePath, 1024, factor, 0); - DFSTestUtil.waitReplication(fs, replFilePath, factor); + // create a contiguous file + Path replDirPath = new Path("/replicated"); + Path replFilePath = new Path(replDirPath, "replfile"); + final short factor = 3; + DFSTestUtil.createFile(fs, replFilePath, 1024, factor, 0); + DFSTestUtil.waitReplication(fs, replFilePath, factor); - // create a large striped file - Path ecDirPath = new Path("/striped"); - Path largeFilePath = new Path(ecDirPath, "largeFile"); - DFSTestUtil.createStripedFile(cluster, largeFilePath, ecDirPath, 1, 2, true); + // create a large striped file + Path ecDirPath = new Path("/striped"); + Path largeFilePath = new Path(ecDirPath, "largeFile"); + DFSTestUtil.createStripedFile(cluster, largeFilePath, ecDirPath, 1, 2, + true); - // create a small striped file - Path smallFilePath = new Path(ecDirPath, "smallFile"); - DFSTestUtil.writeFile(fs, smallFilePath, "hello world!"); + // create a small striped file + Path smallFilePath = new Path(ecDirPath, "smallFile"); + DFSTestUtil.writeFile(fs, smallFilePath, "hello world!"); - long replTime = fs.getFileStatus(replFilePath).getAccessTime(); - long ecTime = fs.getFileStatus(largeFilePath).getAccessTime(); - Thread.sleep(precision); - setupAuditLogs(); - String outStr = runFsck(conf, 0, true, "/"); - verifyAuditLogs(); - assertEquals(replTime, fs.getFileStatus(replFilePath).getAccessTime()); - assertEquals(ecTime, fs.getFileStatus(largeFilePath).getAccessTime()); - System.out.println(outStr); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - if (fs != null) {try{fs.close();} catch(Exception e){}} - cluster.shutdown(); + long replTime = fs.getFileStatus(replFilePath).getAccessTime(); + long ecTime = fs.getFileStatus(largeFilePath).getAccessTime(); + Thread.sleep(precision); + setupAuditLogs(); + String outStr = runFsck(conf, 0, true, "/"); + verifyAuditLogs(); + assertEquals(replTime, fs.getFileStatus(replFilePath).getAccessTime()); + assertEquals(ecTime, fs.getFileStatus(largeFilePath).getAccessTime()); + System.out.println(outStr); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + shutdownCluster(); - // restart the cluster; bring up namenode but not the data nodes - cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(0).format(false).build(); - outStr = runFsck(conf, 1, true, "/"); - // expect the result is corrupt - assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); - System.out.println(outStr); - } finally { - if (fs != null) {try{fs.close();} catch(Exception e){}} - if (cluster != null) { cluster.shutdown(); } - } + // restart the cluster; bring up namenode but not the data nodes + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(0).format(false).build(); + outStr = runFsck(conf, 1, true, "/"); + // expect the result is corrupt + assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); + System.out.println(outStr); } /** @@ -1837,179 +1711,166 @@ public class TestFsck { */ @Test public void testFsckListCorruptSnapshotFiles() throws Exception { - Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000); conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); DistributedFileSystem hdfs = null; - final short REPL_FACTOR = 1; + final short replFactor = 1; - MiniDFSCluster cluster = null; - try { - int numFiles = 3; - int numSnapshots = 0; - cluster = new MiniDFSCluster.Builder(conf).build(); - cluster.waitActive(); - hdfs = cluster.getFileSystem(); - DFSTestUtil util = new DFSTestUtil.Builder(). - setName("testGetCorruptFiles").setNumFiles(numFiles).setMaxLevels(1). - setMaxSize(1024).build(); + int numFiles = 3; + int numSnapshots = 0; + cluster = new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + hdfs = cluster.getFileSystem(); + DFSTestUtil util = new DFSTestUtil.Builder(). + setName("testGetCorruptFiles").setNumFiles(numFiles).setMaxLevels(1). + setMaxSize(1024).build(); - util.createFiles(hdfs, "/corruptData", (short) 1); - final Path fp = new Path("/corruptData/file"); - util.createFile(hdfs, fp, 1024, REPL_FACTOR, 1000L); - numFiles++; - util.waitReplication(hdfs, "/corruptData", (short) 1); + util.createFiles(hdfs, "/corruptData", (short) 1); + final Path fp = new Path("/corruptData/file"); + util.createFile(hdfs, fp, 1024, replFactor, 1000L); + numFiles++; + util.waitReplication(hdfs, "/corruptData", (short) 1); - hdfs.allowSnapshot(new Path("/corruptData")); - hdfs.createSnapshot(new Path("/corruptData"), "mySnapShot"); - numSnapshots = numFiles; + hdfs.allowSnapshot(new Path("/corruptData")); + hdfs.createSnapshot(new Path("/corruptData"), "mySnapShot"); + numSnapshots = numFiles; - String outStr = - runFsck(conf, 0, false, "/corruptData", "-list-corruptfileblocks"); - System.out.println("1. good fsck out: " + outStr); - assertTrue(outStr.contains("has 0 CORRUPT files")); - // delete the blocks - final String bpid = cluster.getNamesystem().getBlockPoolId(); - for (int i=0; i metadataFiles = MiniDFSCluster.getAllBlockMetadataFiles( - data_dir); - if (metadataFiles == null) - continue; - for (File metadataFile : metadataFiles) { - File blockFile = Block.metaToBlockFile(metadataFile); - assertTrue("Cannot remove file.", blockFile.delete()); - assertTrue("Cannot remove file.", metadataFile.delete()); - } + String outStr = + runFsck(conf, 0, false, "/corruptData", "-list-corruptfileblocks"); + System.out.println("1. good fsck out: " + outStr); + assertTrue(outStr.contains("has 0 CORRUPT files")); + // delete the blocks + final String bpid = cluster.getNamesystem().getBlockPoolId(); + for (int i=0; i metadataFiles = MiniDFSCluster.getAllBlockMetadataFiles( + dataDir); + if (metadataFiles == null) { + continue; + } + for (File metadataFile : metadataFiles) { + File blockFile = Block.metaToBlockFile(metadataFile); + assertTrue("Cannot remove file.", blockFile.delete()); + assertTrue("Cannot remove file.", metadataFile.delete()); } } - // Delete file when it has a snapshot - hdfs.delete(fp, false); - numFiles--; - - // wait for the namenode to see the corruption - final NamenodeProtocols namenode = cluster.getNameNodeRpc(); - CorruptFileBlocks corruptFileBlocks = namenode - .listCorruptFileBlocks("/corruptData", null); - int numCorrupt = corruptFileBlocks.getFiles().length; - while (numCorrupt == 0) { - Thread.sleep(1000); - corruptFileBlocks = namenode - .listCorruptFileBlocks("/corruptData", null); - numCorrupt = corruptFileBlocks.getFiles().length; - } - - // with -includeSnapshots all files are reported - outStr = runFsck(conf, -1, true, "/corruptData", - "-list-corruptfileblocks", "-includeSnapshots"); - System.out.println("2. bad fsck include snapshot out: " + outStr); - assertTrue(outStr - .contains("has " + (numFiles + numSnapshots) + " CORRUPT files")); - assertTrue(outStr.contains("/.snapshot/")); - - // without -includeSnapshots only non-snapshots are reported - outStr = - runFsck(conf, -1, true, "/corruptData", "-list-corruptfileblocks"); - System.out.println("3. bad fsck exclude snapshot out: " + outStr); - assertTrue(outStr.contains("has " + numFiles + " CORRUPT files")); - assertFalse(outStr.contains("/.snapshot/")); - } finally { - if (cluster != null) {cluster.shutdown();} } + // Delete file when it has a snapshot + hdfs.delete(fp, false); + numFiles--; + + // wait for the namenode to see the corruption + final NamenodeProtocols namenode = cluster.getNameNodeRpc(); + CorruptFileBlocks corruptFileBlocks = namenode + .listCorruptFileBlocks("/corruptData", null); + int numCorrupt = corruptFileBlocks.getFiles().length; + while (numCorrupt == 0) { + Thread.sleep(1000); + corruptFileBlocks = namenode + .listCorruptFileBlocks("/corruptData", null); + numCorrupt = corruptFileBlocks.getFiles().length; + } + + // with -includeSnapshots all files are reported + outStr = runFsck(conf, -1, true, "/corruptData", + "-list-corruptfileblocks", "-includeSnapshots"); + System.out.println("2. bad fsck include snapshot out: " + outStr); + assertTrue(outStr + .contains("has " + (numFiles + numSnapshots) + " CORRUPT files")); + assertTrue(outStr.contains("/.snapshot/")); + + // without -includeSnapshots only non-snapshots are reported + outStr = + runFsck(conf, -1, true, "/corruptData", "-list-corruptfileblocks"); + System.out.println("3. bad fsck exclude snapshot out: " + outStr); + assertTrue(outStr.contains("has " + numFiles + " CORRUPT files")); + assertFalse(outStr.contains("/.snapshot/")); } @Test (timeout = 300000) public void testFsckMoveAfterCorruption() throws Exception { - final int DFS_BLOCK_SIZE = 512 * 1024; - final int NUM_DATANODES = 1; - final int REPLICATION = 1; - MiniDFSCluster cluster = null; - try { - final Configuration conf = new HdfsConfiguration(); - conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE); - conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L); - conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); - conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, REPLICATION); - cluster = new MiniDFSCluster.Builder(conf).build(); - DistributedFileSystem dfs = cluster.getFileSystem(); - cluster.waitActive(); + final int dfsBlockSize = 512 * 1024; + final int numDatanodes = 1; + final int replication = 1; + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, dfsBlockSize); + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L); + conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replication); + cluster = new MiniDFSCluster.Builder(conf).build(); + DistributedFileSystem dfs = cluster.getFileSystem(); + cluster.waitActive(); - final String srcDir = "/srcdat"; - final DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsck") - .setMinSize(DFS_BLOCK_SIZE * 2).setMaxSize(DFS_BLOCK_SIZE * 3) - .setNumFiles(1).build(); - util.createFiles(dfs, srcDir, (short) REPLICATION); - final String fileNames[] = util.getFileNames(srcDir); - FSImage.LOG.info("Created files: " + Arrays.toString(fileNames)); + final String srcDir = "/srcdat"; + final DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsck") + .setMinSize(dfsBlockSize * 2).setMaxSize(dfsBlockSize * 3) + .setNumFiles(1).build(); + util.createFiles(dfs, srcDir, (short) replication); + final String[] fileNames = util.getFileNames(srcDir); + LOG.info("Created files: " + Arrays.toString(fileNames)); - // Run fsck here. The output is automatically logged for easier debugging - String outStr = runFsck(conf, 0, true, "/", "-files", "-blocks"); - assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + // Run fsck here. The output is automatically logged for easier debugging + String outStr = runFsck(conf, 0, true, "/", "-files", "-blocks"); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); - // Corrupt the first block - final DFSClient dfsClient = new DFSClient( - new InetSocketAddress("localhost", cluster.getNameNodePort()), conf); - final String blockFileToCorrupt = fileNames[0]; - final CorruptedTestFile ctf = new CorruptedTestFile(blockFileToCorrupt, - Sets.newHashSet(0), dfsClient, NUM_DATANODES, DFS_BLOCK_SIZE); - ctf.corruptBlocks(cluster); + // Corrupt the first block + final DFSClient dfsClient = new DFSClient( + new InetSocketAddress("localhost", cluster.getNameNodePort()), conf); + final String blockFileToCorrupt = fileNames[0]; + final CorruptedTestFile ctf = new CorruptedTestFile(blockFileToCorrupt, + Sets.newHashSet(0), dfsClient, numDatanodes, dfsBlockSize); + ctf.corruptBlocks(cluster); - // Wait for fsck to discover all the missing blocks - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - try { - final String str = runFsck(conf, 1, false, "/"); - String numCorrupt = null; - for (String line : str.split(LINE_SEPARATOR)) { - Matcher m = numCorruptBlocksPattern.matcher(line); - if (m.matches()) { - numCorrupt = m.group(1); - break; - } + // Wait for fsck to discover all the missing blocks + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + try { + final String str = runFsck(conf, 1, false, "/"); + String numCorrupt = null; + for (String line : str.split(LINE_SEPARATOR)) { + Matcher m = NUM_CORRUPT_BLOCKS_PATTERN.matcher(line); + if (m.matches()) { + numCorrupt = m.group(1); + break; } - if (numCorrupt == null) { - Assert.fail("Cannot find corrupt blocks count in fsck output."); - } - if (Integer.parseInt(numCorrupt) == ctf.getTotalMissingBlocks()) { - assertTrue(str.contains(NamenodeFsck.CORRUPT_STATUS)); - return true; - } - } catch (Exception e) { - FSImage.LOG.error("Exception caught", e); - Assert.fail("Caught unexpected exception."); } - return false; + if (numCorrupt == null) { + Assert.fail("Cannot find corrupt blocks count in fsck output."); + } + if (Integer.parseInt(numCorrupt) == ctf.getTotalMissingBlocks()) { + assertTrue(str.contains(NamenodeFsck.CORRUPT_STATUS)); + return true; + } + } catch (Exception e) { + LOG.error("Exception caught", e); + Assert.fail("Caught unexpected exception."); } - }, 1000, 60000); - - runFsck(conf, 1, true, "/", "-files", "-blocks", "-racks"); - FSImage.LOG.info("Moving blocks to lost+found"); - // Fsck will return error since we corrupted a block - runFsck(conf, 1, false, "/", "-move"); - - final List retVal = new ArrayList<>(); - final RemoteIterator iter = - dfs.listFiles(new Path("/lost+found"), true); - while (iter.hasNext()) { - retVal.add(iter.next()); + return false; } - FSImage.LOG.info("Items in lost+found: " + retVal); + }, 1000, 60000); - // Expect all good blocks moved, only corrupted block skipped. - long totalLength = 0; - for (LocatedFileStatus lfs: retVal) { - totalLength += lfs.getLen(); - } - Assert.assertTrue("Nothing is moved to lost+found!", totalLength > 0); - util.cleanup(dfs, srcDir); - } finally { - if (cluster != null) { - cluster.shutdown(); - } + runFsck(conf, 1, true, "/", "-files", "-blocks", "-racks"); + LOG.info("Moving blocks to lost+found"); + // Fsck will return error since we corrupted a block + runFsck(conf, 1, false, "/", "-move"); + + final List retVal = new ArrayList<>(); + final RemoteIterator iter = + dfs.listFiles(new Path("/lost+found"), true); + while (iter.hasNext()) { + retVal.add(iter.next()); } + LOG.info("Items in lost+found: " + retVal); + + // Expect all good blocks moved, only corrupted block skipped. + long totalLength = 0; + for (LocatedFileStatus lfs: retVal) { + totalLength += lfs.getLen(); + } + Assert.assertTrue("Nothing is moved to lost+found!", totalLength > 0); + util.cleanup(dfs, srcDir); } @Test(timeout = 60000) @@ -2030,7 +1891,6 @@ public class TestFsck { final String[] racks = {"/rack1"}; final String[] hosts = {"127.0.0.1"}; HostsFileWriter hostsFileWriter = new HostsFileWriter(); - Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replFactor); if (defineUpgradeDomain) { @@ -2039,7 +1899,6 @@ public class TestFsck { hostsFileWriter.initialize(conf, "temp/fsckupgradedomain"); } - MiniDFSCluster cluster; DistributedFileSystem dfs; cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDN). hosts(hosts).racks(racks).build(); @@ -2074,9 +1933,135 @@ public class TestFsck { if (defineUpgradeDomain) { hostsFileWriter.cleanup(); } - if (cluster != null) { - cluster.shutdown(); - } } } + + @Test (timeout = 300000) + public void testFsckCorruptECFile() throws Exception { + DistributedFileSystem fs = null; + int dataBlocks = ErasureCodingPolicyManager + .getSystemDefaultPolicy().getNumDataUnits(); + int parityBlocks = ErasureCodingPolicyManager + .getSystemDefaultPolicy().getNumParityUnits(); + int cellSize = ErasureCodingPolicyManager + .getSystemDefaultPolicy().getCellSize(); + int totalSize = dataBlocks + parityBlocks; + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(totalSize).build(); + fs = cluster.getFileSystem(); + Map dnIndices = new HashMap<>(); + ArrayList dnList = cluster.getDataNodes(); + for (int i = 0; i < totalSize; i++) { + dnIndices.put(dnList.get(i).getIpcPort(), i); + } + + // create file + Path ecDirPath = new Path("/striped"); + fs.mkdir(ecDirPath, FsPermission.getDirDefault()); + fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null); + Path file = new Path(ecDirPath, "corrupted"); + final int length = cellSize * dataBlocks; + final byte[] bytes = StripedFileTestUtil.generateBytes(length); + DFSTestUtil.writeFile(fs, file, bytes); + + LocatedStripedBlock lsb = (LocatedStripedBlock)fs.getClient() + .getLocatedBlocks(file.toString(), 0, cellSize * dataBlocks).get(0); + final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(lsb, + cellSize, dataBlocks, parityBlocks); + + // make an unrecoverable ec file with corrupted blocks + for(int i = 0; i < parityBlocks + 1; i++) { + int ipcPort = blks[i].getLocations()[0].getIpcPort(); + int dnIndex = dnIndices.get(ipcPort); + File storageDir = cluster.getInstanceStorageDir(dnIndex, 0); + File blkFile = MiniDFSCluster.getBlockFile(storageDir, + blks[i].getBlock()); + Assert.assertTrue("Block file does not exist", blkFile.exists()); + + FileOutputStream out = new FileOutputStream(blkFile); + out.write("corruption".getBytes()); + } + + // disable the heart beat from DN so that the corrupted block record is + // kept in NameNode + for (DataNode dn : cluster.getDataNodes()) { + DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true); + } + + // Read the file to trigger reportBadBlocks + try { + IOUtils.copyBytes(fs.open(file), new IOUtils.NullOutputStream(), conf, + true); + } catch (IOException ie) { + assertTrue(ie.getMessage().contains( + "missingChunksNum=" + (parityBlocks + 1))); + } + + waitForUnrecoverableBlockGroup(conf); + + String outStr = runFsck(conf, 1, true, "/"); + assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); + } + + @Test (timeout = 300000) + public void testFsckMissingECFile() throws Exception { + DistributedFileSystem fs = null; + int dataBlocks = ErasureCodingPolicyManager + .getSystemDefaultPolicy().getNumDataUnits(); + int parityBlocks = ErasureCodingPolicyManager + .getSystemDefaultPolicy().getNumParityUnits(); + int cellSize = ErasureCodingPolicyManager + .getSystemDefaultPolicy().getCellSize(); + int totalSize = dataBlocks + parityBlocks; + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(totalSize).build(); + fs = cluster.getFileSystem(); + + // create file + Path ecDirPath = new Path("/striped"); + fs.mkdir(ecDirPath, FsPermission.getDirDefault()); + fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null); + Path file = new Path(ecDirPath, "missing"); + final int length = cellSize * dataBlocks; + final byte[] bytes = StripedFileTestUtil.generateBytes(length); + DFSTestUtil.writeFile(fs, file, bytes); + + // make an unrecoverable ec file with missing blocks + ArrayList dns = cluster.getDataNodes(); + DatanodeID dnId; + for (int i = 0; i < parityBlocks + 1; i++) { + dnId = dns.get(i).getDatanodeId(); + cluster.stopDataNode(dnId.getXferAddr()); + cluster.setDataNodeDead(dnId); + } + + waitForUnrecoverableBlockGroup(conf); + + String outStr = runFsck(conf, 1, true, "/", "-files", "-blocks", + "-locations"); + assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); + assertTrue(outStr.contains("Live_repl=" + (dataBlocks - 1))); + } + + private void waitForUnrecoverableBlockGroup(Configuration configuration) + throws TimeoutException, InterruptedException { + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + try { + ByteArrayOutputStream bStream = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bStream, true); + ToolRunner.run(new DFSck(configuration, out), new String[] {"/"}); + String outStr = bStream.toString(); + if (outStr.contains("UNRECOVERABLE BLOCK GROUPS")) { + return true; + } + } catch (Exception e) { + LOG.error("Exception caught", e); + Assert.fail("Caught unexpected exception."); + } + return false; + } + }, 1000, 60000); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java index 26efce5b75c..d7a2c811a59 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java @@ -55,6 +55,7 @@ import org.apache.hadoop.ipc.RpcConstants; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.LightWeightCache; import org.junit.After; @@ -111,19 +112,33 @@ public class TestNamenodeRetryCache { } } + static class DummyCall extends Server.Call { + private UserGroupInformation ugi; + + DummyCall(int callId, byte[] clientId) { + super(callId, 1, null, null, RpcKind.RPC_PROTOCOL_BUFFER, clientId); + try { + ugi = UserGroupInformation.getCurrentUser(); + } catch (IOException ioe) { + } + } + @Override + public UserGroupInformation getRemoteUser() { + return ugi; + } + } /** Set the current Server RPC call */ public static void newCall() { - Server.Call call = new Server.Call(++callId, 1, null, null, - RpcKind.RPC_PROTOCOL_BUFFER, CLIENT_ID); + Server.Call call = new DummyCall(++callId, CLIENT_ID); Server.getCurCall().set(call); } public static void resetCall() { - Server.Call call = new Server.Call(RpcConstants.INVALID_CALL_ID, 1, null, - null, RpcKind.RPC_PROTOCOL_BUFFER, RpcConstants.DUMMY_CLIENT_ID); + Server.Call call = new DummyCall(RpcConstants.INVALID_CALL_ID, + RpcConstants.DUMMY_CLIENT_ID); Server.getCurCall().set(call); } - + private void concatSetup(String file1, String file2) throws Exception { DFSTestUtil.createFile(filesystem, new Path(file1), BlockSize, (short)1, 0L); DFSTestUtil.createFile(filesystem, new Path(file2), BlockSize, (short)1, 0L); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java index 24ec1a28379..07f01d00ad6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.io.FileNotFoundException; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -30,12 +31,15 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.Mockito; /** Test snapshot related operations. */ public class TestSnapshotPathINodes { @@ -166,6 +170,9 @@ public class TestSnapshotPathINodes { assertEquals(sub1.toString(), nodesInPath.getPath(2)); assertEquals(file1.toString(), nodesInPath.getPath(3)); + assertEquals(file1.getParent().toString(), + nodesInPath.getParentINodesInPath().getPath()); + nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false); assertEquals(nodesInPath.length(), components.length); assertSnapshot(nodesInPath, false, null, -1); @@ -212,6 +219,9 @@ public class TestSnapshotPathINodes { // The number of INodes returned should still be components.length // since we put a null in the inode array for ".snapshot" assertEquals(nodesInPath.length(), components.length); + // ensure parent inodes can strip the .snapshot + assertEquals(sub1.toString(), + nodesInPath.getParentINodesInPath().getPath()); // No SnapshotRoot dir is included in the resolved inodes assertSnapshot(nodesInPath, true, snapshot, -1); @@ -420,4 +430,16 @@ public class TestSnapshotPathINodes { hdfs.deleteSnapshot(sub1, "s3"); hdfs.disallowSnapshot(sub1); } + + @Test + public void testShortCircuitSnapshotSearch() throws SnapshotException { + FSNamesystem fsn = cluster.getNamesystem(); + SnapshotManager sm = fsn.getSnapshotManager(); + assertEquals(0, sm.getNumSnapshottableDirs()); + + INodesInPath iip = Mockito.mock(INodesInPath.class); + List snapDirs = new ArrayList<>(); + FSDirSnapshotOp.checkSnapshot(fsn.getFSDirectory(), iip, snapDirs); + Mockito.verifyZeroInteractions(iip); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java index 8d68b15bd43..3a910c14aa3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java @@ -31,9 +31,12 @@ import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.io.retry.MultiException; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; +import org.apache.log4j.Level; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; @@ -47,6 +50,11 @@ public class TestRequestHedgingProxyProvider { private URI nnUri; private String ns; + @BeforeClass + public static void setupClass() throws Exception { + GenericTestUtils.setLogLevel(RequestHedgingProxyProvider.LOG, Level.TRACE); + } + @Before public void setup() throws URISyntaxException { ns = "mycluster-" + Time.monotonicNow(); @@ -66,13 +74,19 @@ public class TestRequestHedgingProxyProvider { @Test public void testHedgingWhenOneFails() throws Exception { final NamenodeProtocols goodMock = Mockito.mock(NamenodeProtocols.class); - Mockito.when(goodMock.getStats()).thenReturn(new long[] {1}); + Mockito.when(goodMock.getStats()).thenAnswer(new Answer() { + @Override + public long[] answer(InvocationOnMock invocation) throws Throwable { + Thread.sleep(1000); + return new long[]{1}; + } + }); final NamenodeProtocols badMock = Mockito.mock(NamenodeProtocols.class); Mockito.when(badMock.getStats()).thenThrow(new IOException("Bad mock !!")); RequestHedgingProxyProvider provider = new RequestHedgingProxyProvider<>(conf, nnUri, NamenodeProtocols.class, - createFactory(goodMock, badMock)); + createFactory(badMock, goodMock)); long[] stats = provider.getProxy().proxy.getStats(); Assert.assertTrue(stats.length == 1); Mockito.verify(badMock).getStats(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java new file mode 100644 index 00000000000..4d3a4f030e2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.metrics; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.namenode.top.TopConf; +import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.lib.Interns; +import org.junit.Test; + +import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * Test for MetricsSource part of the {@link TopMetrics} impl. + */ +public class TestTopMetrics { + @Test + public void testPresence() { + Configuration conf = new Configuration(); + TopConf topConf = new TopConf(conf); + TopMetrics topMetrics = new TopMetrics(conf, + topConf.nntopReportingPeriodsMs); + // Dummy command + topMetrics.report("test", "listStatus"); + topMetrics.report("test", "listStatus"); + topMetrics.report("test", "listStatus"); + + MetricsRecordBuilder rb = getMetrics(topMetrics); + MetricsCollector mc = rb.parent(); + + verify(mc).addRecord(TOPMETRICS_METRICS_SOURCE_NAME + ".windowMs=60000"); + verify(mc).addRecord(TOPMETRICS_METRICS_SOURCE_NAME + ".windowMs=300000"); + verify(mc).addRecord(TOPMETRICS_METRICS_SOURCE_NAME + ".windowMs=1500000"); + + verify(rb, times(3)).addCounter(Interns.info("op=listStatus.TotalCount", + "Total operation count"), 3L); + verify(rb, times(3)).addCounter(Interns.info("op=*.TotalCount", + "Total operation count"), 3L); + + verify(rb, times(3)).addCounter(Interns.info("op=listStatus." + + "user=test.count", "Total operations performed by user"), 3L); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java index 91eec7884d9..d1b3aa6f870 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java @@ -36,8 +36,10 @@ import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -2411,4 +2413,201 @@ public class TestRenameWithSnapshots { assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file2", "newDir/file2")); assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file3", "newDir/file1")); } + + private void checkSpaceConsumed(String message, Path directory, + long expectedSpace) throws Exception { + ContentSummary summary = hdfs.getContentSummary(directory); + assertEquals(message, expectedSpace, summary.getSpaceConsumed()); + } + + /** + * Runs through various combinations of renames, deletes, appends and other + * operations in a snapshotted directory and ensures disk usage summaries + * (e.g. du -s) are computed correctly. + * + * @throws Exception + */ + @Test (timeout=300000) + public void testDu() throws Exception { + File tempFile = File.createTempFile("testDu-", ".tmp"); + tempFile.deleteOnExit(); + + final FileSystem localfs = FileSystem.getLocal(conf); + final Path localOriginal = new Path(tempFile.getPath()); + final Path dfsRoot = new Path("/testDu"); + final Path dfsOriginal = new Path(dfsRoot, "original"); + final Path dfsRenamed1 = new Path(dfsRoot, "renamed1"); + final Path dfsRenamed2 = new Path(dfsRoot, "renamed2"); + final Path dfsAppended = new Path(dfsRoot, "appended"); + + /* We will test with a single block worth of data. If we don't at least use + a multiple of BLOCKSIZE, append operations will modify snapshotted blocks + and other factors will come into play here that we'll have to account for */ + final long spaceIncrement = BLOCKSIZE * REPL; + final byte[] appendData = new byte[(int) BLOCKSIZE]; + DFSTestUtil.createFile(localfs, localOriginal, BLOCKSIZE, REPL, SEED); + + FSDataOutputStream out = null; + long expectedSpace = 0; + + hdfs.mkdirs(dfsRoot); + checkSpaceConsumed("Du is wrong immediately", + dfsRoot, 0L); + + hdfs.copyFromLocalFile(localOriginal, dfsOriginal); + expectedSpace += spaceIncrement; + checkSpaceConsumed("Du is wrong after creating / copying file", + dfsRoot, expectedSpace); + + SnapshotTestHelper.createSnapshot(hdfs, dfsRoot, "s0"); + checkSpaceConsumed("Du is wrong after snapshotting", + dfsRoot, expectedSpace); + + hdfs.rename(dfsOriginal, dfsRenamed1); + checkSpaceConsumed("Du is wrong after 1 rename", + dfsRoot, expectedSpace); + + hdfs.rename(dfsRenamed1, dfsRenamed2); + checkSpaceConsumed("Du is wrong after 2 renames", + dfsRoot, expectedSpace); + + hdfs.delete(dfsRenamed2, false); + checkSpaceConsumed("Du is wrong after deletion", + dfsRoot, expectedSpace); + + hdfs.copyFromLocalFile(localOriginal, dfsOriginal); + expectedSpace += spaceIncrement; + checkSpaceConsumed("Du is wrong after replacing a renamed file", + dfsRoot, expectedSpace); + + hdfs.copyFromLocalFile(localOriginal, dfsAppended); + expectedSpace += spaceIncrement; + SnapshotTestHelper.createSnapshot(hdfs, dfsRoot, "s1"); + + out = hdfs.append(dfsAppended); + out.write(appendData); + out.close(); + expectedSpace += spaceIncrement; + checkSpaceConsumed("Du is wrong after 1 snapshot + append", + dfsRoot, expectedSpace); + + SnapshotTestHelper.createSnapshot(hdfs, dfsRoot, "s2"); + out = hdfs.append(dfsAppended); + out.write(appendData); + out.close(); + expectedSpace += spaceIncrement; + checkSpaceConsumed("Du is wrong after 2 snapshot + appends", + dfsRoot, expectedSpace); + + SnapshotTestHelper.createSnapshot(hdfs, dfsRoot, "s3"); + out = hdfs.append(dfsAppended); + out.write(appendData); + out.close(); + expectedSpace += spaceIncrement; + hdfs.rename(dfsAppended, dfsRenamed1); + checkSpaceConsumed("Du is wrong after snapshot, append, & rename", + dfsRoot, expectedSpace); + hdfs.delete(dfsRenamed1, false); + // everything but the last append is snapshotted + expectedSpace -= spaceIncrement; + checkSpaceConsumed("Du is wrong after snapshot, append, delete & rename", + dfsRoot, expectedSpace); + + hdfs.delete(dfsOriginal, false); + hdfs.deleteSnapshot(dfsRoot, "s0"); + hdfs.deleteSnapshot(dfsRoot, "s1"); + hdfs.deleteSnapshot(dfsRoot, "s2"); + hdfs.deleteSnapshot(dfsRoot, "s3"); + expectedSpace = 0; + checkSpaceConsumed("Du is wrong after deleting all files and snapshots", + dfsRoot, expectedSpace); + } + + /** + * Runs through various combinations of renames, deletes, appends and other + * operations between two snapshotted directories and ensures disk usage + * summaries (e.g. du -s) are computed correctly. + * + * This test currently assumes some incorrect behavior when files have been + * moved between subdirectories of the one being queried. In the cases + * below, only 1 block worth of data should ever actually be used. However + * if there are 2 - 3 subdirectories that do contained or have contained + * when snapshotted the same file, that file will be counted 2-3 times, + * respectively, since each directory is computed independently recursively. + * + * @throws Exception + */ + @Test (timeout=300000) + public void testDuMultipleDirs() throws Exception { + File tempFile = File.createTempFile("testDuMultipleDirs-", "" + ".tmp"); + tempFile.deleteOnExit(); + + final FileSystem localfs = FileSystem.getLocal(conf); + final Path localOriginal = new Path(tempFile.getPath()); + final Path dfsRoot = new Path("/testDuMultipleDirs"); + final Path snapshottable1 = new Path(dfsRoot, "snapshottable1"); + final Path snapshottable2 = new Path(dfsRoot, "snapshottable2"); + final Path nonsnapshottable = new Path(dfsRoot, "nonsnapshottable"); + final Path subdirectory = new Path(snapshottable1, "subdirectory"); + final Path dfsOriginal = new Path(snapshottable1, "file"); + final Path renamedNonsnapshottable = new Path(nonsnapshottable, "file"); + final Path renamedSnapshottable = new Path(snapshottable2, "file"); + final Path renamedSubdirectory = new Path(subdirectory, "file"); + + /* We will test with a single block worth of data. If we don't at least use + a multiple of BLOCKSIZE, append operations will modify snapshotted blocks + and other factors will come into play here that we'll have to account for */ + final long spaceConsumed = BLOCKSIZE * REPL; + DFSTestUtil.createFile(localfs, localOriginal, BLOCKSIZE, REPL, SEED); + + hdfs.mkdirs(snapshottable1); + hdfs.mkdirs(snapshottable2); + hdfs.mkdirs(nonsnapshottable); + hdfs.mkdirs(subdirectory); + checkSpaceConsumed("Du is wrong immediately", + dfsRoot, 0L); + + hdfs.copyFromLocalFile(localOriginal, dfsOriginal); + checkSpaceConsumed("Du is wrong after creating / copying file", + snapshottable1, spaceConsumed); + + SnapshotTestHelper.createSnapshot(hdfs, snapshottable1, "s1"); + checkSpaceConsumed("Du is wrong in original dir after 1st snapshot", + snapshottable1, spaceConsumed); + + hdfs.rename(dfsOriginal, renamedNonsnapshottable); + checkSpaceConsumed("Du is wrong in original dir after 1st rename", + snapshottable1, spaceConsumed); + checkSpaceConsumed("Du is wrong in non-snapshottable dir after 1st rename", + nonsnapshottable, spaceConsumed); + checkSpaceConsumed("Du is wrong in root dir after 1st rename", + dfsRoot, spaceConsumed); + + hdfs.rename(renamedNonsnapshottable, renamedSnapshottable); + checkSpaceConsumed("Du is wrong in original dir after 2nd rename", + snapshottable1, spaceConsumed); + checkSpaceConsumed("Du is wrong in non-snapshottable dir after 2nd rename", + nonsnapshottable, 0); + checkSpaceConsumed("Du is wrong in snapshottable dir after 2nd rename", + snapshottable2, spaceConsumed); + checkSpaceConsumed("Du is wrong in root dir after 2nd rename", + dfsRoot, spaceConsumed); + + SnapshotTestHelper.createSnapshot(hdfs, snapshottable2, "s2"); + hdfs.rename(renamedSnapshottable, renamedSubdirectory); + checkSpaceConsumed("Du is wrong in original dir after 3rd rename", + snapshottable1, spaceConsumed); + checkSpaceConsumed("Du is wrong in snapshottable dir after 3rd rename", + snapshottable2, spaceConsumed); + checkSpaceConsumed("Du is wrong in original subdirectory after 3rd rename", + subdirectory, spaceConsumed); + checkSpaceConsumed("Du is wrong in root dir after 3rd rename", + dfsRoot, spaceConsumed); + + hdfs.delete(renamedSubdirectory, false); + hdfs.deleteSnapshot(snapshottable1, "s1"); + hdfs.deleteSnapshot(snapshottable2, "s2"); + checkSpaceConsumed("Du is wrong after deleting all files and snapshots", + dfsRoot, 0); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java index 15e1c04b817..604bf791d59 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode.web.resources; import static org.mockito.Mockito.*; import java.io.IOException; +import java.net.InetAddress; import java.util.Arrays; import java.util.List; @@ -62,6 +63,9 @@ public class TestWebHdfsDataLocality { private static final String RACK1 = "/rack1"; private static final String RACK2 = "/rack2"; + private static final String LOCALHOST = + InetAddress.getLoopbackAddress().getHostName(); + @Rule public final ExpectedException exception = ExpectedException.none(); @@ -96,7 +100,8 @@ public class TestWebHdfsDataLocality { //The chosen datanode must be the same as the client address final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, PutOpParam.Op.CREATE, -1L, blocksize, null); + namenode, f, PutOpParam.Op.CREATE, -1L, blocksize, null, + LOCALHOST); Assert.assertEquals(ipAddr, chosen.getIpAddr()); } } @@ -121,19 +126,22 @@ public class TestWebHdfsDataLocality { { //test GETFILECHECKSUM final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize, null); + namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize, null, + LOCALHOST); Assert.assertEquals(expected, chosen); } { //test OPEN final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, GetOpParam.Op.OPEN, 0, blocksize, null); + namenode, f, GetOpParam.Op.OPEN, 0, blocksize, null, + LOCALHOST); Assert.assertEquals(expected, chosen); } { //test APPEND final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, PostOpParam.Op.APPEND, -1L, blocksize, null); + namenode, f, PostOpParam.Op.APPEND, -1L, blocksize, null, + LOCALHOST); Assert.assertEquals(expected, chosen); } } finally { @@ -189,7 +197,7 @@ public class TestWebHdfsDataLocality { { // test GETFILECHECKSUM final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize, - sb.toString()); + sb.toString(), LOCALHOST); for (int j = 0; j <= i; j++) { Assert.assertNotEquals(locations[j].getHostName(), chosen.getHostName()); @@ -198,7 +206,8 @@ public class TestWebHdfsDataLocality { { // test OPEN final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode( - namenode, f, GetOpParam.Op.OPEN, 0, blocksize, sb.toString()); + namenode, f, GetOpParam.Op.OPEN, 0, blocksize, sb.toString(), + LOCALHOST); for (int j = 0; j <= i; j++) { Assert.assertNotEquals(locations[j].getHostName(), chosen.getHostName()); @@ -208,7 +217,7 @@ public class TestWebHdfsDataLocality { { // test APPEND final DatanodeInfo chosen = NamenodeWebHdfsMethods .chooseDatanode(namenode, f, PostOpParam.Op.APPEND, -1L, - blocksize, sb.toString()); + blocksize, sb.toString(), LOCALHOST); for (int j = 0; j <= i; j++) { Assert.assertNotEquals(locations[j].getHostName(), chosen.getHostName()); @@ -229,6 +238,6 @@ public class TestWebHdfsDataLocality { exception.expect(IOException.class); exception.expectMessage("Namesystem has not been intialized yet."); NamenodeWebHdfsMethods.chooseDatanode(nn, "/path", PutOpParam.Op.CREATE, 0, - DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT, null); + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT, null, LOCALHOST); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java index ac14438c633..8e217c22192 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java @@ -34,6 +34,7 @@ import java.util.Iterator; import java.util.Map; import java.util.concurrent.TimeoutException; +import org.apache.commons.collections.map.LinkedMap; import org.apache.commons.lang.mutable.MutableBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -502,8 +503,8 @@ public class TestShortCircuitCache { public void visit(int numOutstandingMmaps, Map replicas, Map failedLoads, - Map evictable, - Map evictableMmapped) { + LinkedMap evictable, + LinkedMap evictableMmapped) { ShortCircuitReplica replica = replicas.get( ExtendedBlockId.fromExtendedBlock(block)); Assert.assertNotNull(replica); @@ -518,8 +519,8 @@ public class TestShortCircuitCache { public void visit(int numOutstandingMmaps, Map replicas, Map failedLoads, - Map evictable, - Map evictableMmapped) { + LinkedMap evictable, + LinkedMap evictableMmapped) { ShortCircuitReplica replica = replicas.get( ExtendedBlockId.fromExtendedBlock(block)); Assert.assertNotNull(replica); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index e71c5cc526e..dca42ea9c06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.tools; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY; @@ -30,12 +31,15 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.ReconfigurationUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.PathUtils; +import org.apache.hadoop.util.ToolRunner; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -68,28 +72,53 @@ public class TestDFSAdmin { private DFSAdmin admin; private DataNode datanode; private NameNode namenode; + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private final ByteArrayOutputStream err = new ByteArrayOutputStream(); + private static final PrintStream OLD_OUT = System.out; + private static final PrintStream OLD_ERR = System.err; @Before public void setUp() throws Exception { conf = new Configuration(); + conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 3); restartCluster(); admin = new DFSAdmin(); } + private void redirectStream() { + System.setOut(new PrintStream(out)); + System.setErr(new PrintStream(err)); + } + + private void resetStream() { + out.reset(); + err.reset(); + } + @After public void tearDown() throws Exception { + try { + System.out.flush(); + System.err.flush(); + } finally { + System.setOut(OLD_OUT); + System.setErr(OLD_ERR); + } + if (cluster != null) { cluster.shutdown(); cluster = null; } + + resetStream(); } private void restartCluster() throws IOException { if (cluster != null) { cluster.shutdown(); } - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); cluster.waitActive(); datanode = cluster.getDataNodes().get(0); namenode = cluster.getNameNode(); @@ -111,28 +140,92 @@ public class TestDFSAdmin { String nodeType, String address, final List outs, final List errs) throws IOException { ByteArrayOutputStream bufOut = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bufOut); + PrintStream outStream = new PrintStream(bufOut); ByteArrayOutputStream bufErr = new ByteArrayOutputStream(); - PrintStream err = new PrintStream(bufErr); + PrintStream errStream = new PrintStream(bufErr); if (methodName.equals("getReconfigurableProperties")) { - admin.getReconfigurableProperties(nodeType, address, out, err); + admin.getReconfigurableProperties( + nodeType, + address, + outStream, + errStream); } else if (methodName.equals("getReconfigurationStatus")) { - admin.getReconfigurationStatus(nodeType, address, out, err); + admin.getReconfigurationStatus(nodeType, address, outStream, errStream); } else if (methodName.equals("startReconfiguration")) { - admin.startReconfiguration(nodeType, address, out, err); + admin.startReconfiguration(nodeType, address, outStream, errStream); } - Scanner scanner = new Scanner(bufOut.toString()); + scanIntoList(bufOut, outs); + scanIntoList(bufErr, errs); + } + + private static void scanIntoList( + final ByteArrayOutputStream baos, + final List list) { + final Scanner scanner = new Scanner(baos.toString()); while (scanner.hasNextLine()) { - outs.add(scanner.nextLine()); + list.add(scanner.nextLine()); } scanner.close(); - scanner = new Scanner(bufErr.toString()); - while (scanner.hasNextLine()) { - errs.add(scanner.nextLine()); + } + + @Test(timeout = 30000) + public void testGetDatanodeInfo() throws Exception { + redirectStream(); + final DFSAdmin dfsAdmin = new DFSAdmin(conf); + + for (int i = 0; i < cluster.getDataNodes().size(); i++) { + resetStream(); + final DataNode dn = cluster.getDataNodes().get(i); + final String addr = String.format( + "%s:%d", + dn.getXferAddress().getHostString(), + dn.getIpcPort()); + final int ret = ToolRunner.run(dfsAdmin, + new String[]{"-getDatanodeInfo", addr}); + assertEquals(0, ret); + + /* collect outputs */ + final List outs = Lists.newArrayList(); + scanIntoList(out, outs); + /* verify results */ + assertEquals( + "One line per DataNode like: Uptime: XXX, Software version: x.y.z," + + " Config version: core-x.y.z,hdfs-x", + 1, outs.size()); + assertThat(outs.get(0), + is(allOf(containsString("Uptime:"), + containsString("Software version"), + containsString("Config version")))); + } + } + + /** + * Test that if datanode is not reachable, some DFSAdmin commands will fail + * elegantly with non-zero ret error code along with exception error message. + */ + @Test(timeout = 60000) + public void testDFSAdminUnreachableDatanode() throws Exception { + redirectStream(); + final DFSAdmin dfsAdmin = new DFSAdmin(conf); + for (String command : new String[]{"-getDatanodeInfo", + "-evictWriters", "-getBalancerBandwidth"}) { + // Connecting to Xfer port instead of IPC port will get + // Datanode unreachable. java.io.EOFException + final String dnDataAddr = datanode.getXferAddress().getHostString() + ":" + + datanode.getXferPort(); + resetStream(); + final List outs = Lists.newArrayList(); + final int ret = ToolRunner.run(dfsAdmin, + new String[]{command, dnDataAddr}); + assertEquals(-1, ret); + + scanIntoList(out, outs); + assertTrue("Unexpected " + command + " stdout: " + out, outs.isEmpty()); + assertTrue("Unexpected " + command + " stderr: " + err, + err.toString().contains("Exception")); } - scanner.close(); } @Test(timeout = 30000) @@ -261,6 +354,55 @@ public class TestDFSAdmin { }, 100, 100 * 100); } + @Test(timeout = 30000) + public void testPrintTopology() throws Exception { + redirectStream(); + + /* init conf */ + final Configuration dfsConf = new HdfsConfiguration(); + final File baseDir = new File( + PathUtils.getTestDir(getClass()), + GenericTestUtils.getMethodName()); + dfsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); + + final int numDn = 4; + final String[] racks = { + "/d1/r1", "/d1/r2", + "/d2/r1", "/d2/r2"}; + + /* init cluster using topology */ + try (MiniDFSCluster miniCluster = new MiniDFSCluster.Builder(dfsConf) + .numDataNodes(numDn).racks(racks).build()) { + + miniCluster.waitActive(); + assertEquals(numDn, miniCluster.getDataNodes().size()); + final DFSAdmin dfsAdmin = new DFSAdmin(dfsConf); + + resetStream(); + final int ret = ToolRunner.run(dfsAdmin, new String[] {"-printTopology"}); + + /* collect outputs */ + final List outs = Lists.newArrayList(); + scanIntoList(out, outs); + + /* verify results */ + assertEquals(0, ret); + assertEquals( + "There should be three lines per Datanode: the 1st line is" + + " rack info, 2nd node info, 3rd empty line. The total" + + " should be as a result of 3 * numDn.", + 12, outs.size()); + assertThat(outs.get(0), + is(allOf(containsString("Rack:"), containsString("/d1/r1")))); + assertThat(outs.get(3), + is(allOf(containsString("Rack:"), containsString("/d1/r2")))); + assertThat(outs.get(6), + is(allOf(containsString("Rack:"), containsString("/d2/r1")))); + assertThat(outs.get(9), + is(allOf(containsString("Rack:"), containsString("/d2/r2")))); + } + } + @Test(timeout = 30000) public void testNameNodeGetReconfigurationStatus() throws IOException, InterruptedException, TimeoutException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml index ddd4adc4453..0294368754f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml @@ -388,5 +388,95 @@ + + + Test success of getFileEncryptionInfo on a EZ file + + -fs NAMENODE -mkdir /src + -createZone -path /src -keyName myKey + -fs NAMENODE -touchz /src/file + -getFileEncryptionInfo -path /src/file + + + -fs NAMENODE -rm -r /src + + + + SubstringComparator + keyName: myKey, ezKeyVersionName: myKey@0 + + + + + + Test failure of getFileEncryptionInfo on a non-EZ file + + -fs NAMENODE -mkdir /src + -fs NAMENODE -touchz /src/cleartext + -getFileEncryptionInfo -path /src/cleartext + + + -fs NAMENODE -rm -r /src + + + + SubstringComparator + No FileEncryptionInfo found for path + + + + + + Test failure of getFileEncryptionInfo on a non-exist file + + -getFileEncryptionInfo -path /src/file + + + -fs NAMENODE -rm -r /src + + + + SubstringComparator + FileNotFoundException: + + + + + + Test failure of getFileEncryptionInfo on a EZ dir + + -fs NAMENODE -mkdir /src + -createZone -path /src -keyName myKey + -getFileEncryptionInfo -path /src + + + -fs NAMENODE -rm -r /src + + + + SubstringComparator + No FileEncryptionInfo found for path + + + + + + Test failure of getFileEncryptionInfo on a EZ subdir + + -fs NAMENODE -mkdir /src + -createZone -path /src -keyName myKey + -fs NAMENODE -mkdir /src/dir + -getFileEncryptionInfo -path /src/dir + + + -fs NAMENODE -rm -r /src + + + + SubstringComparator + No FileEncryptionInfo found for path + + + diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml index 4bb0545d8f1..a0993c9c0a8 100644 --- a/hadoop-hdfs-project/pom.xml +++ b/hadoop-hdfs-project/pom.xml @@ -35,7 +35,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdfs-client hadoop-hdfs-native-client hadoop-hdfs-httpfs - hadoop-hdfs/src/contrib/bkjournal hadoop-hdfs-nfs diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index d94f8a55b0e..4a8a90e7abf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -149,7 +149,6 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; import org.apache.hadoop.yarn.util.Clock; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.log4j.LogManager; @@ -1303,44 +1302,77 @@ public class MRAppMaster extends CompositeService { } private void processRecovery() throws IOException{ - if (appAttemptID.getAttemptId() == 1) { - return; // no need to recover on the first attempt + boolean attemptRecovery = shouldAttemptRecovery(); + boolean recoverySucceeded = true; + if (attemptRecovery) { + LOG.info("Attempting to recover."); + try { + parsePreviousJobHistory(); + } catch (IOException e) { + LOG.warn("Unable to parse prior job history, aborting recovery", e); + recoverySucceeded = false; + } + } + + if (!isFirstAttempt() && (!attemptRecovery || !recoverySucceeded)) { + amInfos.addAll(readJustAMInfos()); + } + } + + private boolean isFirstAttempt() { + return appAttemptID.getAttemptId() == 1; + } + + /** + * Check if the current job attempt should try to recover from previous + * job attempts if any. + */ + private boolean shouldAttemptRecovery() throws IOException { + if (isFirstAttempt()) { + return false; // no need to recover on the first attempt } boolean recoveryEnabled = getConfig().getBoolean( MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE_DEFAULT); + if (!recoveryEnabled) { + LOG.info("Not attempting to recover. Recovery disabled. To enable " + + "recovery, set " + MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE); + return false; + } boolean recoverySupportedByCommitter = isRecoverySupported(); + if (!recoverySupportedByCommitter) { + LOG.info("Not attempting to recover. Recovery is not supported by " + + committer.getClass() + ". Use an OutputCommitter that supports" + + " recovery."); + return false; + } - // If a shuffle secret was not provided by the job client then this app - // attempt will generate one. However that disables recovery if there - // are reducers as the shuffle secret would be app attempt specific. - int numReduceTasks = getConfig().getInt(MRJobConfig.NUM_REDUCES, 0); + int reducerCount = getConfig().getInt(MRJobConfig.NUM_REDUCES, 0); + + // If a shuffle secret was not provided by the job client, one will be + // generated in this job attempt. However, that disables recovery if + // there are reducers as the shuffle secret would be job attempt specific. boolean shuffleKeyValidForRecovery = TokenCache.getShuffleSecretKey(jobCredentials) != null; - - if (recoveryEnabled && recoverySupportedByCommitter - && (numReduceTasks <= 0 || shuffleKeyValidForRecovery)) { - LOG.info("Recovery is enabled. " - + "Will try to recover from previous life on best effort basis."); - try { - parsePreviousJobHistory(); - } catch (IOException e) { - LOG.warn("Unable to parse prior job history, aborting recovery", e); - // try to get just the AMInfos - amInfos.addAll(readJustAMInfos()); - } - } else { - LOG.info("Will not try to recover. recoveryEnabled: " - + recoveryEnabled + " recoverySupportedByCommitter: " - + recoverySupportedByCommitter + " numReduceTasks: " - + numReduceTasks + " shuffleKeyValidForRecovery: " - + shuffleKeyValidForRecovery + " ApplicationAttemptID: " - + appAttemptID.getAttemptId()); - // Get the amInfos anyways whether recovery is enabled or not - amInfos.addAll(readJustAMInfos()); + if (reducerCount > 0 && !shuffleKeyValidForRecovery) { + LOG.info("Not attempting to recover. The shuffle key is invalid for " + + "recovery."); + return false; } + + // If the intermediate data is encrypted, recovering the job requires the + // access to the key. Until the encryption key is persisted, we should + // avoid attempts to recover. + boolean spillEncrypted = CryptoUtils.isEncryptedSpillEnabled(getConfig()); + if (reducerCount > 0 && spillEncrypted) { + LOG.info("Not attempting to recover. Intermediate spill encryption" + + " is enabled."); + return false; + } + + return true; } private static FSDataInputStream getPreviousJobHistoryStream( @@ -1440,6 +1472,10 @@ public class MRAppMaster extends CompositeService { return amInfos; } + public boolean recovered() { + return recoveredJobStartTime > 0; + } + /** * This can be overridden to instantiate multiple jobs and create a * workflow. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java index 9d5f0ae4d7e..071575a156a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java @@ -579,6 +579,72 @@ public class TestRecovery { app.verifyCompleted(); } + @Test + public void testRecoveryWithSpillEncryption() throws Exception { + int runCount = 0; + MRApp app = new MRAppWithHistory(1, 1, false, this.getClass().getName(), + true, ++runCount) { + }; + Configuration conf = new Configuration(); + conf.setBoolean(MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, true); + conf.setBoolean("mapred.mapper.new-api", true); + conf.setBoolean("mapred.reducer.new-api", true); + conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); + conf.set(FileOutputFormat.OUTDIR, outputDir.toString()); + conf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true); + + // run the MR job at the first attempt + Job jobAttempt1 = app.submit(conf); + app.waitForState(jobAttempt1, JobState.RUNNING); + + Iterator tasks = jobAttempt1.getTasks().values().iterator(); + + // finish the map task but the reduce task + Task mapper = tasks.next(); + app.waitForState(mapper, TaskState.RUNNING); + TaskAttempt mapAttempt = mapper.getAttempts().values().iterator().next(); + app.waitForState(mapAttempt, TaskAttemptState.RUNNING); + app.getContext().getEventHandler().handle( + new TaskAttemptEvent(mapAttempt.getID(), TaskAttemptEventType.TA_DONE)); + app.waitForState(mapper, TaskState.SUCCEEDED); + + // crash the first attempt of the MR job + app.stop(); + + // run the MR job again at the second attempt + app = new MRAppWithHistory(1, 1, false, this.getClass().getName(), false, + ++runCount); + Job jobAttempt2 = app.submit(conf); + Assert.assertTrue("Recovery from previous job attempt is processed even " + + "though intermediate data encryption is enabled.", !app.recovered()); + + // The map task succeeded from previous job attempt will not be recovered + // because the data spill encryption is enabled. + // Let's finish the job at the second attempt and verify its completion. + app.waitForState(jobAttempt2, JobState.RUNNING); + tasks = jobAttempt2.getTasks().values().iterator(); + mapper = tasks.next(); + Task reducer = tasks.next(); + + // finish the map task first + app.waitForState(mapper, TaskState.RUNNING); + mapAttempt = mapper.getAttempts().values().iterator().next(); + app.waitForState(mapAttempt, TaskAttemptState.RUNNING); + app.getContext().getEventHandler().handle( + new TaskAttemptEvent(mapAttempt.getID(), TaskAttemptEventType.TA_DONE)); + app.waitForState(mapper, TaskState.SUCCEEDED); + + // then finish the reduce task + TaskAttempt redAttempt = reducer.getAttempts().values().iterator().next(); + app.waitForState(redAttempt, TaskAttemptState.RUNNING); + app.getContext().getEventHandler().handle( + new TaskAttemptEvent(redAttempt.getID(), TaskAttemptEventType.TA_DONE)); + app.waitForState(reducer, TaskState.SUCCEEDED); + + // verify that the job succeeds at the 2rd attempt + app.waitForState(jobAttempt2, JobState.SUCCEEDED); + } + /** * This test case primarily verifies if the recovery is controlled through config * property. In this case, recover is turned OFF. AM with 3 maps and 0 reduce. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java index acb31bd1b46..21d37c82c08 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java @@ -247,9 +247,11 @@ public class TestAMWebApp { HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection(); conn.setInstanceFollowRedirects(false); conn.connect(); - String expectedURL = - scheme + conf.get(YarnConfiguration.PROXY_ADDRESS) - + ProxyUriUtils.getPath(app.getAppID(), "/mapreduce"); + + // Because we're not calling from the proxy's address, we'll be redirected + String expectedURL = scheme + conf.get(YarnConfiguration.PROXY_ADDRESS) + + ProxyUriUtils.getPath(app.getAppID(), "/mapreduce", true); + Assert.assertEquals(expectedURL, conn.getHeaderField(HttpHeaders.LOCATION)); Assert.assertEquals(HttpStatus.SC_MOVED_TEMPORARILY, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 57164045578..1325b743a10 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -505,7 +505,7 @@ public interface MRJobConfig { */ public static final String MR_CLIENT_JOB_MAX_RETRIES = MR_PREFIX + "client.job.max-retries"; - public static final int DEFAULT_MR_CLIENT_JOB_MAX_RETRIES = 0; + public static final int DEFAULT_MR_CLIENT_JOB_MAX_RETRIES = 3; /** * How long to wait between jobclient retries on failure diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 73aaa7a7db7..fe292126210 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1505,12 +1505,12 @@ yarn.app.mapreduce.client.job.max-retries - 0 + 3 The number of retries the client will make for getJob and - dependent calls. The default is 0 as this is generally only needed for - non-HDFS DFS where additional, high level retries are required to avoid - spurious failures during the getJob call. 30 is a good value for - WASB + dependent calls. + This is needed for non-HDFS DFS where additional, high level + retries are required to avoid spurious failures during the getJob call. + 30 is a good value for WASB diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java index 4895a5b036b..e02232d2b2f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java @@ -225,10 +225,10 @@ public class JobClientUnitTest { //To prevent the test from running for a very long time, lower the retry JobConf conf = new JobConf(); - conf.set(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, "3"); + conf.setInt(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, 2); TestJobClientGetJob client = new TestJobClientGetJob(conf); - JobID id = new JobID("ajob",1); + JobID id = new JobID("ajob", 1); RunningJob rj = mock(RunningJob.class); client.setRunningJob(rj); @@ -236,13 +236,35 @@ public class JobClientUnitTest { assertNotNull(client.getJob(id)); assertEquals(client.getLastGetJobRetriesCounter(), 0); - //3 retry - client.setGetJobRetries(3); + //2 retries + client.setGetJobRetries(2); assertNotNull(client.getJob(id)); - assertEquals(client.getLastGetJobRetriesCounter(), 3); + assertEquals(client.getLastGetJobRetriesCounter(), 2); - //beyond MAPREDUCE_JOBCLIENT_GETJOB_MAX_RETRY_KEY, will get null - client.setGetJobRetries(5); + //beyond yarn.app.mapreduce.client.job.max-retries, will get null + client.setGetJobRetries(3); + assertNull(client.getJob(id)); + } + + @Test + public void testGetJobRetryDefault() throws Exception { + + //To prevent the test from running for a very long time, lower the retry + JobConf conf = new JobConf(); + + TestJobClientGetJob client = new TestJobClientGetJob(conf); + JobID id = new JobID("ajob", 1); + RunningJob rj = mock(RunningJob.class); + client.setRunningJob(rj); + + //3 retries (default) + client.setGetJobRetries(MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES); + assertNotNull(client.getJob(id)); + assertEquals(client.getLastGetJobRetriesCounter(), + MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES); + + //beyond yarn.app.mapreduce.client.job.max-retries, will get null + client.setGetJobRetries(MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES + 1); assertNull(client.getJob(id)); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java index 22fe3443275..7fbb22af636 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java @@ -246,6 +246,9 @@ public class TeraGen extends Configured implements Tool { private static void usage() throws IOException { System.err.println("teragen "); + System.err.println("If you want to generate data and store them as " + + "erasure code striping file, just make sure that the parent dir " + + "of has erasure code policy set"); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java index fd3ea78fdcd..73c446d7e7e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java @@ -20,6 +20,8 @@ package org.apache.hadoop.examples.terasort; import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -40,6 +42,7 @@ import org.apache.hadoop.mapreduce.security.TokenCache; * An output format that writes the key and value appended together. */ public class TeraOutputFormat extends FileOutputFormat { + private static final Log LOG = LogFactory.getLog(TeraOutputFormat.class); private OutputCommitter committer = null; /** @@ -74,10 +77,22 @@ public class TeraOutputFormat extends FileOutputFormat { out.write(key.getBytes(), 0, key.getLength()); out.write(value.getBytes(), 0, value.getLength()); } - + public void close(TaskAttemptContext context) throws IOException { if (finalSync) { - out.hsync(); + try { + out.hsync(); + } catch (UnsupportedOperationException e) { + /* + * Currently, hsync operation on striping file with erasure code + * policy is not supported yet. So this is a workaround to make + * teragen and terasort to support directory with striping files. In + * future, if the hsync operation is supported on striping file, this + * workaround should be removed. + */ + LOG.info("Operation hsync is not supported so far on path with " + + "erasure code policy set"); + } } out.close(); } @@ -135,5 +150,4 @@ public class TeraOutputFormat extends FileOutputFormat { } return committer; } - } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java index 9beff3e92c1..040d13ffb6f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java @@ -287,6 +287,9 @@ public class TeraSort extends Configured implements Tool { for (TeraSortConfigKeys teraSortConfigKeys : TeraSortConfigKeys.values()) { System.err.println(teraSortConfigKeys.toString()); } + System.err.println("If you want to store the output data as " + + "erasure code striping file, just make sure that the parent dir " + + "of has erasure code policy set"); } public int run(String[] args) throws Exception { diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java index cd2651bb8ba..f6faea088b6 100644 --- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java @@ -160,7 +160,7 @@ public class VersionInfoMojo extends AbstractMojo { if (index > -1) { res[0] = path.substring(0, index - 1); int branchIndex = index + "branches".length() + 1; - index = path.indexOf("/", branchIndex); + index = path.indexOf('/', branchIndex); if (index > -1) { res[1] = path.substring(branchIndex, index); } else { diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index bf4fac74bbe..4423d94bf30 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -87,22 +87,6 @@ - - org.apache.maven.plugins - maven-source-plugin - - - prepare-package - - jar - test-jar - - - - - true - - org.codehaus.mojo findbugs-maven-plugin @@ -116,7 +100,6 @@ org.apache.maven.plugins maven-javadoc-plugin - true 512m true false diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index d9a01a07616..82adebf7968 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -114,7 +114,7 @@ 2.5 2.4 2.3 - 2.7 + 2.9 1.2 1.5 1.9 @@ -437,6 +437,12 @@ ${project.version} + + org.apache.hadoop + hadoop-aliyun + ${project.version} + + org.apache.hadoop hadoop-kms @@ -929,12 +935,6 @@ - - org.apache.bookkeeper - bookkeeper-server - 4.2.3 - compile - org.hsqldb hsqldb @@ -1005,6 +1005,22 @@ 4.2.0 + + com.aliyun.oss + aliyun-sdk-oss + 2.2.1 + + + org.apache.httpcomponents + httpclient + + + commons-beanutils + commons-beanutils + + + + xerces xercesImpl diff --git a/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml new file mode 100644 index 00000000000..40d78d0cd6c --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml @@ -0,0 +1,18 @@ + + + diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml new file mode 100644 index 00000000000..358b18b1c45 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/pom.xml @@ -0,0 +1,154 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-alpha2-SNAPSHOT + ../../hadoop-project + + hadoop-aliyun + Apache Hadoop Aliyun OSS support + jar + + + UTF-8 + true + + + + + tests-off + + + src/test/resources/auth-keys.xml + + + + true + + + + tests-on + + + src/test/resources/auth-keys.xml + + + + false + + + + + + + + org.codehaus.mojo + findbugs-maven-plugin + + true + true + ${basedir}/dev-support/findbugs-exclude.xml + + Max + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + + false + false + + + + org.apache.maven.plugins + maven-surefire-plugin + + 3600 + + + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt + + + + + + + + + + junit + junit + test + + + + com.aliyun.oss + aliyun-sdk-oss + compile + + + + org.apache.hadoop + hadoop-common + compile + + + + org.apache.hadoop + hadoop-common + test + test-jar + + + org.apache.hadoop + hadoop-distcp + test + + + org.apache.hadoop + hadoop-distcp + test + test-jar + + + org.apache.hadoop + hadoop-yarn-server-tests + test + test-jar + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + test + + + diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java new file mode 100644 index 00000000000..b46c67aa5e7 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import com.aliyun.oss.common.auth.Credentials; +import com.aliyun.oss.common.auth.CredentialsProvider; +import com.aliyun.oss.common.auth.DefaultCredentials; +import com.aliyun.oss.common.auth.InvalidCredentialsException; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +/** + * Support session credentials for authenticating with Aliyun. + */ +public class AliyunCredentialsProvider implements CredentialsProvider { + private Credentials credentials = null; + + public AliyunCredentialsProvider(Configuration conf) + throws IOException { + String accessKeyId; + String accessKeySecret; + String securityToken; + try { + accessKeyId = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_ID); + accessKeySecret = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_SECRET); + } catch (IOException e) { + throw new InvalidCredentialsException(e); + } + + try { + securityToken = AliyunOSSUtils.getValueWithKey(conf, SECURITY_TOKEN); + } catch (IOException e) { + securityToken = null; + } + + if (StringUtils.isEmpty(accessKeyId) + || StringUtils.isEmpty(accessKeySecret)) { + throw new InvalidCredentialsException( + "AccessKeyId and AccessKeySecret should not be null or empty."); + } + + if (StringUtils.isNotEmpty(securityToken)) { + credentials = new DefaultCredentials(accessKeyId, accessKeySecret, + securityToken); + } else { + credentials = new DefaultCredentials(accessKeyId, accessKeySecret); + } + } + + @Override + public void setCredentials(Credentials creds) { + if (creds == null) { + throw new InvalidCredentialsException("Credentials should not be null."); + } + + credentials = creds; + } + + @Override + public Credentials getCredentials() { + if (credentials == null) { + throw new InvalidCredentialsException("Invalid credentials"); + } + + return credentials; + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java new file mode 100644 index 00000000000..3b266c83934 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -0,0 +1,580 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Progressable; + +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; +import com.aliyun.oss.model.ObjectMetadata; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +/** + * Implementation of {@link FileSystem} for + * Aliyun OSS, used to access OSS blob system in a filesystem style. + */ +public class AliyunOSSFileSystem extends FileSystem { + private static final Logger LOG = + LoggerFactory.getLogger(AliyunOSSFileSystem.class); + private URI uri; + private String bucket; + private Path workingDir; + private AliyunOSSFileSystemStore store; + private int maxKeys; + + @Override + public FSDataOutputStream append(Path path, int bufferSize, + Progressable progress) throws IOException { + throw new IOException("Append is not supported!"); + } + + @Override + public void close() throws IOException { + try { + store.close(); + } finally { + super.close(); + } + } + + @Override + public FSDataOutputStream create(Path path, FsPermission permission, + boolean overwrite, int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { + String key = pathToKey(path); + FileStatus status = null; + + try { + // get the status or throw a FNFE + status = getFileStatus(path); + + // if the thread reaches here, there is something at the path + if (status.isDirectory()) { + // path references a directory + throw new FileAlreadyExistsException(path + " is a directory"); + } + if (!overwrite) { + // path references a file and overwrite is disabled + throw new FileAlreadyExistsException(path + " already exists"); + } + LOG.debug("Overwriting file {}", path); + } catch (FileNotFoundException e) { + // this means the file is not found + } + + return new FSDataOutputStream(new AliyunOSSOutputStream(getConf(), + store, key, progress, statistics), (Statistics)(null)); + } + + @Override + public boolean delete(Path path, boolean recursive) throws IOException { + try { + return innerDelete(getFileStatus(path), recursive); + } catch (FileNotFoundException e) { + LOG.debug("Couldn't delete {} - does not exist", path); + return false; + } + } + + /** + * Delete an object. See {@link #delete(Path, boolean)}. + * + * @param status fileStatus object + * @param recursive if path is a directory and set to + * true, the directory is deleted else throws an exception. In + * case of a file the recursive can be set to either true or false. + * @return true if delete is successful else false. + * @throws IOException due to inability to delete a directory or file. + */ + private boolean innerDelete(FileStatus status, boolean recursive) + throws IOException { + Path f = status.getPath(); + String p = f.toUri().getPath(); + FileStatus[] statuses; + // indicating root directory "/". + if (p.equals("/")) { + statuses = listStatus(status.getPath()); + boolean isEmptyDir = statuses.length <= 0; + return rejectRootDirectoryDelete(isEmptyDir, recursive); + } + + String key = pathToKey(f); + if (status.isDirectory()) { + if (!recursive) { + // Check whether it is an empty directory or not + statuses = listStatus(status.getPath()); + if (statuses.length > 0) { + throw new IOException("Cannot remove directory " + f + + ": It is not empty!"); + } else { + // Delete empty directory without '-r' + key = AliyunOSSUtils.maybeAddTrailingSlash(key); + store.deleteObject(key); + } + } else { + store.deleteDirs(key); + } + } else { + store.deleteObject(key); + } + + createFakeDirectoryIfNecessary(f); + return true; + } + + /** + * Implements the specific logic to reject root directory deletion. + * The caller must return the result of this call, rather than + * attempt to continue with the delete operation: deleting root + * directories is never allowed. This method simply implements + * the policy of when to return an exit code versus raise an exception. + * @param isEmptyDir empty directory or not + * @param recursive recursive flag from command + * @return a return code for the operation + * @throws PathIOException if the operation was explicitly rejected. + */ + private boolean rejectRootDirectoryDelete(boolean isEmptyDir, + boolean recursive) throws IOException { + LOG.info("oss delete the {} root directory of {}", bucket, recursive); + if (isEmptyDir) { + return true; + } + if (recursive) { + return false; + } else { + // reject + throw new PathIOException(bucket, "Cannot delete root path"); + } + } + + private void createFakeDirectoryIfNecessary(Path f) throws IOException { + String key = pathToKey(f); + if (StringUtils.isNotEmpty(key) && !exists(f)) { + LOG.debug("Creating new fake directory at {}", f); + mkdir(pathToKey(f.getParent())); + } + } + + @Override + public FileStatus getFileStatus(Path path) throws IOException { + Path qualifiedPath = path.makeQualified(uri, workingDir); + String key = pathToKey(qualifiedPath); + + // Root always exists + if (key.length() == 0) { + return new FileStatus(0, true, 1, 0, 0, qualifiedPath); + } + + ObjectMetadata meta = store.getObjectMetadata(key); + // If key not found and key does not end with "/" + if (meta == null && !key.endsWith("/")) { + // In case of 'dir + "/"' + key += "/"; + meta = store.getObjectMetadata(key); + } + if (meta == null) { + ObjectListing listing = store.listObjects(key, 1, null, false); + if (CollectionUtils.isNotEmpty(listing.getObjectSummaries()) || + CollectionUtils.isNotEmpty(listing.getCommonPrefixes())) { + return new FileStatus(0, true, 1, 0, 0, qualifiedPath); + } else { + throw new FileNotFoundException(path + ": No such file or directory!"); + } + } else if (objectRepresentsDirectory(key, meta.getContentLength())) { + return new FileStatus(0, true, 1, 0, 0, qualifiedPath); + } else { + return new FileStatus(meta.getContentLength(), false, 1, + getDefaultBlockSize(path), meta.getLastModified().getTime(), + qualifiedPath); + } + } + + @Override + public String getScheme() { + return "oss"; + } + + @Override + public URI getUri() { + return uri; + } + + @Override + public Path getWorkingDirectory() { + return workingDir; + } + + @Deprecated + public long getDefaultBlockSize() { + return getConf().getLong(FS_OSS_BLOCK_SIZE_KEY, FS_OSS_BLOCK_SIZE_DEFAULT); + } + + @Override + public String getCanonicalServiceName() { + // Does not support Token + return null; + } + + /** + * Initialize new FileSystem. + * + * @param name the uri of the file system, including host, port, etc. + * @param conf configuration of the file system + * @throws IOException IO problems + */ + public void initialize(URI name, Configuration conf) throws IOException { + super.initialize(name, conf); + + bucket = name.getHost(); + uri = java.net.URI.create(name.getScheme() + "://" + name.getAuthority()); + workingDir = new Path("/user", + System.getProperty("user.name")).makeQualified(uri, null); + + store = new AliyunOSSFileSystemStore(); + store.initialize(name, conf, statistics); + maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT); + setConf(conf); + } + + /** + * Check if OSS object represents a directory. + * + * @param name object key + * @param size object content length + * @return true if object represents a directory + */ + private boolean objectRepresentsDirectory(final String name, + final long size) { + return StringUtils.isNotEmpty(name) && name.endsWith("/") && size == 0L; + } + + /** + * Turn a path (relative or otherwise) into an OSS key. + * + * @param path the path of the file. + * @return the key of the object that represents the file. + */ + private String pathToKey(Path path) { + if (!path.isAbsolute()) { + path = new Path(workingDir, path); + } + + return path.toUri().getPath().substring(1); + } + + private Path keyToPath(String key) { + return new Path("/" + key); + } + + @Override + public FileStatus[] listStatus(Path path) throws IOException { + String key = pathToKey(path); + if (LOG.isDebugEnabled()) { + LOG.debug("List status for path: " + path); + } + + final List result = new ArrayList(); + final FileStatus fileStatus = getFileStatus(path); + + if (fileStatus.isDirectory()) { + if (LOG.isDebugEnabled()) { + LOG.debug("listStatus: doing listObjects for directory " + key); + } + + ObjectListing objects = store.listObjects(key, maxKeys, null, false); + while (true) { + statistics.incrementReadOps(1); + for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { + String objKey = objectSummary.getKey(); + if (objKey.equals(key + "/")) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignoring: " + objKey); + } + continue; + } else { + Path keyPath = keyToPath(objectSummary.getKey()) + .makeQualified(uri, workingDir); + if (LOG.isDebugEnabled()) { + LOG.debug("Adding: fi: " + keyPath); + } + result.add(new FileStatus(objectSummary.getSize(), false, 1, + getDefaultBlockSize(keyPath), + objectSummary.getLastModified().getTime(), keyPath)); + } + } + + for (String prefix : objects.getCommonPrefixes()) { + if (prefix.equals(key + "/")) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignoring: " + prefix); + } + continue; + } else { + Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir); + if (LOG.isDebugEnabled()) { + LOG.debug("Adding: rd: " + keyPath); + } + result.add(new FileStatus(0, true, 1, 0, 0, keyPath)); + } + } + + if (objects.isTruncated()) { + if (LOG.isDebugEnabled()) { + LOG.debug("listStatus: list truncated - getting next batch"); + } + String nextMarker = objects.getNextMarker(); + objects = store.listObjects(key, maxKeys, nextMarker, false); + statistics.incrementReadOps(1); + } else { + break; + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Adding: rd (not a dir): " + path); + } + result.add(fileStatus); + } + + return result.toArray(new FileStatus[result.size()]); + } + + /** + * Used to create an empty file that represents an empty directory. + * + * @param key directory path + * @return true if directory is successfully created + * @throws IOException + */ + private boolean mkdir(final String key) throws IOException { + String dirName = key; + if (StringUtils.isNotEmpty(key)) { + if (!key.endsWith("/")) { + dirName += "/"; + } + store.storeEmptyFile(dirName); + } + return true; + } + + @Override + public boolean mkdirs(Path path, FsPermission permission) + throws IOException { + try { + FileStatus fileStatus = getFileStatus(path); + + if (fileStatus.isDirectory()) { + return true; + } else { + throw new FileAlreadyExistsException("Path is a file: " + path); + } + } catch (FileNotFoundException e) { + validatePath(path); + String key = pathToKey(path); + return mkdir(key); + } + } + + /** + * Check whether the path is a valid path. + * + * @param path the path to be checked. + * @throws IOException + */ + private void validatePath(Path path) throws IOException { + Path fPart = path.getParent(); + do { + try { + FileStatus fileStatus = getFileStatus(fPart); + if (fileStatus.isDirectory()) { + // If path exists and a directory, exit + break; + } else { + throw new FileAlreadyExistsException(String.format( + "Can't make directory for path '%s', it is a file.", fPart)); + } + } catch (FileNotFoundException fnfe) { + } + fPart = fPart.getParent(); + } while (fPart != null); + } + + @Override + public FSDataInputStream open(Path path, int bufferSize) throws IOException { + final FileStatus fileStatus = getFileStatus(path); + if (fileStatus.isDirectory()) { + throw new FileNotFoundException("Can't open " + path + + " because it is a directory"); + } + + return new FSDataInputStream(new AliyunOSSInputStream(getConf(), store, + pathToKey(path), fileStatus.getLen(), statistics)); + } + + @Override + public boolean rename(Path srcPath, Path dstPath) throws IOException { + if (srcPath.isRoot()) { + // Cannot rename root of file system + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot rename the root of a filesystem"); + } + return false; + } + Path parent = dstPath.getParent(); + while (parent != null && !srcPath.equals(parent)) { + parent = parent.getParent(); + } + if (parent != null) { + return false; + } + FileStatus srcStatus = getFileStatus(srcPath); + FileStatus dstStatus; + try { + dstStatus = getFileStatus(dstPath); + } catch (FileNotFoundException fnde) { + dstStatus = null; + } + if (dstStatus == null) { + // If dst doesn't exist, check whether dst dir exists or not + dstStatus = getFileStatus(dstPath.getParent()); + if (!dstStatus.isDirectory()) { + throw new IOException(String.format( + "Failed to rename %s to %s, %s is a file", srcPath, dstPath, + dstPath.getParent())); + } + } else { + if (srcStatus.getPath().equals(dstStatus.getPath())) { + return !srcStatus.isDirectory(); + } else if (dstStatus.isDirectory()) { + // If dst is a directory + dstPath = new Path(dstPath, srcPath.getName()); + FileStatus[] statuses; + try { + statuses = listStatus(dstPath); + } catch (FileNotFoundException fnde) { + statuses = null; + } + if (statuses != null && statuses.length > 0) { + // If dst exists and not a directory / not empty + throw new FileAlreadyExistsException(String.format( + "Failed to rename %s to %s, file already exists or not empty!", + srcPath, dstPath)); + } + } else { + // If dst is not a directory + throw new FileAlreadyExistsException(String.format( + "Failed to rename %s to %s, file already exists!", srcPath, + dstPath)); + } + } + if (srcStatus.isDirectory()) { + copyDirectory(srcPath, dstPath); + } else { + copyFile(srcPath, dstPath); + } + + return srcPath.equals(dstPath) || delete(srcPath, true); + } + + /** + * Copy file from source path to destination path. + * (the caller should make sure srcPath is a file and dstPath is valid) + * + * @param srcPath source path. + * @param dstPath destination path. + * @return true if file is successfully copied. + */ + private boolean copyFile(Path srcPath, Path dstPath) { + String srcKey = pathToKey(srcPath); + String dstKey = pathToKey(dstPath); + return store.copyFile(srcKey, dstKey); + } + + /** + * Copy a directory from source path to destination path. + * (the caller should make sure srcPath is a directory, and dstPath is valid) + * + * @param srcPath source path. + * @param dstPath destination path. + * @return true if directory is successfully copied. + */ + private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException { + String srcKey = AliyunOSSUtils + .maybeAddTrailingSlash(pathToKey(srcPath)); + String dstKey = AliyunOSSUtils + .maybeAddTrailingSlash(pathToKey(dstPath)); + + if (dstKey.startsWith(srcKey)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot rename a directory to a subdirectory of self"); + } + return false; + } + + store.storeEmptyFile(dstKey); + ObjectListing objects = store.listObjects(srcKey, maxKeys, null, true); + statistics.incrementReadOps(1); + // Copy files from src folder to dst + while (true) { + for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { + String newKey = + dstKey.concat(objectSummary.getKey().substring(srcKey.length())); + store.copyFile(objectSummary.getKey(), newKey); + } + if (objects.isTruncated()) { + String nextMarker = objects.getNextMarker(); + objects = store.listObjects(srcKey, maxKeys, nextMarker, true); + statistics.incrementReadOps(1); + } else { + break; + } + } + return true; + } + + @Override + public void setWorkingDirectory(Path dir) { + this.workingDir = dir; + } + + public AliyunOSSFileSystemStore getStore() { + return store; + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java new file mode 100644 index 00000000000..9792a7816c3 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java @@ -0,0 +1,516 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.aliyun.oss; + +import com.aliyun.oss.ClientConfiguration; +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.common.auth.CredentialsProvider; +import com.aliyun.oss.common.comm.Protocol; +import com.aliyun.oss.model.AbortMultipartUploadRequest; +import com.aliyun.oss.model.CannedAccessControlList; +import com.aliyun.oss.model.CompleteMultipartUploadRequest; +import com.aliyun.oss.model.CompleteMultipartUploadResult; +import com.aliyun.oss.model.CopyObjectResult; +import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.GetObjectRequest; +import com.aliyun.oss.model.InitiateMultipartUploadRequest; +import com.aliyun.oss.model.InitiateMultipartUploadResult; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.ObjectListing; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.PartETag; +import com.aliyun.oss.model.PutObjectResult; +import com.aliyun.oss.model.UploadPartCopyRequest; +import com.aliyun.oss.model.UploadPartCopyResult; +import com.aliyun.oss.model.UploadPartRequest; +import com.aliyun.oss.model.UploadPartResult; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +/** + * Core implementation of Aliyun OSS Filesystem for Hadoop. + * Provides the bridging logic between Hadoop's abstract filesystem and + * Aliyun OSS. + */ +public class AliyunOSSFileSystemStore { + public static final Logger LOG = + LoggerFactory.getLogger(AliyunOSSFileSystemStore.class); + private FileSystem.Statistics statistics; + private OSSClient ossClient; + private String bucketName; + private long uploadPartSize; + private long multipartThreshold; + private long partSize; + private int maxKeys; + private String serverSideEncryptionAlgorithm; + + public void initialize(URI uri, Configuration conf, + FileSystem.Statistics stat) throws IOException { + statistics = stat; + ClientConfiguration clientConf = new ClientConfiguration(); + clientConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS_KEY, + MAXIMUM_CONNECTIONS_DEFAULT)); + boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS_KEY, + SECURE_CONNECTIONS_DEFAULT); + clientConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); + clientConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES_KEY, + MAX_ERROR_RETRIES_DEFAULT)); + clientConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT_KEY, + ESTABLISH_TIMEOUT_DEFAULT)); + clientConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT_KEY, + SOCKET_TIMEOUT_DEFAULT)); + + String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, ""); + int proxyPort = conf.getInt(PROXY_PORT_KEY, -1); + if (StringUtils.isNotEmpty(proxyHost)) { + clientConf.setProxyHost(proxyHost); + if (proxyPort >= 0) { + clientConf.setProxyPort(proxyPort); + } else { + if (secureConnections) { + LOG.warn("Proxy host set without port. Using HTTPS default 443"); + clientConf.setProxyPort(443); + } else { + LOG.warn("Proxy host set without port. Using HTTP default 80"); + clientConf.setProxyPort(80); + } + } + String proxyUsername = conf.getTrimmed(PROXY_USERNAME_KEY); + String proxyPassword = conf.getTrimmed(PROXY_PASSWORD_KEY); + if ((proxyUsername == null) != (proxyPassword == null)) { + String msg = "Proxy error: " + PROXY_USERNAME_KEY + " or " + + PROXY_PASSWORD_KEY + " set without the other."; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + clientConf.setProxyUsername(proxyUsername); + clientConf.setProxyPassword(proxyPassword); + clientConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN_KEY)); + clientConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION_KEY)); + } else if (proxyPort >= 0) { + String msg = "Proxy error: " + PROXY_PORT_KEY + " set without " + + PROXY_HOST_KEY; + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + + String endPoint = conf.getTrimmed(ENDPOINT_KEY, ""); + CredentialsProvider provider = + AliyunOSSUtils.getCredentialsProvider(conf); + ossClient = new OSSClient(endPoint, provider, clientConf); + uploadPartSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, + MULTIPART_UPLOAD_SIZE_DEFAULT); + multipartThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, + MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT); + partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY, + MULTIPART_UPLOAD_SIZE_DEFAULT); + if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) { + partSize = MIN_MULTIPART_UPLOAD_PART_SIZE; + } + serverSideEncryptionAlgorithm = + conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, ""); + + if (uploadPartSize < 5 * 1024 * 1024) { + LOG.warn(MULTIPART_UPLOAD_SIZE_KEY + " must be at least 5 MB"); + uploadPartSize = 5 * 1024 * 1024; + } + + if (multipartThreshold < 5 * 1024 * 1024) { + LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be at least 5 MB"); + multipartThreshold = 5 * 1024 * 1024; + } + + if (multipartThreshold > 1024 * 1024 * 1024) { + LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be less than 1 GB"); + multipartThreshold = 1024 * 1024 * 1024; + } + + String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT); + if (StringUtils.isNotEmpty(cannedACLName)) { + CannedAccessControlList cannedACL = + CannedAccessControlList.valueOf(cannedACLName); + ossClient.setBucketAcl(bucketName, cannedACL); + } + + maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT); + bucketName = uri.getHost(); + } + + /** + * Delete an object, and update write operation statistics. + * + * @param key key to blob to delete. + */ + public void deleteObject(String key) { + ossClient.deleteObject(bucketName, key); + statistics.incrementWriteOps(1); + } + + /** + * Delete a list of keys, and update write operation statistics. + * + * @param keysToDelete collection of keys to delete. + */ + public void deleteObjects(List keysToDelete) { + if (CollectionUtils.isNotEmpty(keysToDelete)) { + DeleteObjectsRequest deleteRequest = + new DeleteObjectsRequest(bucketName); + deleteRequest.setKeys(keysToDelete); + ossClient.deleteObjects(deleteRequest); + statistics.incrementWriteOps(keysToDelete.size()); + } + } + + /** + * Delete a directory from Aliyun OSS. + * + * @param key directory key to delete. + */ + public void deleteDirs(String key) { + key = AliyunOSSUtils.maybeAddTrailingSlash(key); + ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); + listRequest.setPrefix(key); + listRequest.setDelimiter(null); + listRequest.setMaxKeys(maxKeys); + + while (true) { + ObjectListing objects = ossClient.listObjects(listRequest); + statistics.incrementReadOps(1); + List keysToDelete = new ArrayList(); + for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { + keysToDelete.add(objectSummary.getKey()); + } + deleteObjects(keysToDelete); + if (objects.isTruncated()) { + listRequest.setMarker(objects.getNextMarker()); + } else { + break; + } + } + } + + /** + * Return metadata of a given object key. + * + * @param key object key. + * @return return null if key does not exist. + */ + public ObjectMetadata getObjectMetadata(String key) { + try { + return ossClient.getObjectMetadata(bucketName, key); + } catch (OSSException osse) { + return null; + } finally { + statistics.incrementReadOps(1); + } + } + + /** + * Upload an empty file as an OSS object, using single upload. + * + * @param key object key. + * @throws IOException if failed to upload object. + */ + public void storeEmptyFile(String key) throws IOException { + ObjectMetadata dirMeta = new ObjectMetadata(); + byte[] buffer = new byte[0]; + ByteArrayInputStream in = new ByteArrayInputStream(buffer); + dirMeta.setContentLength(0); + try { + ossClient.putObject(bucketName, key, in, dirMeta); + } finally { + in.close(); + } + } + + /** + * Copy an object from source key to destination key. + * + * @param srcKey source key. + * @param dstKey destination key. + * @return true if file is successfully copied. + */ + public boolean copyFile(String srcKey, String dstKey) { + ObjectMetadata objectMeta = + ossClient.getObjectMetadata(bucketName, srcKey); + long contentLength = objectMeta.getContentLength(); + if (contentLength <= multipartThreshold) { + return singleCopy(srcKey, dstKey); + } else { + return multipartCopy(srcKey, contentLength, dstKey); + } + } + + /** + * Use single copy to copy an OSS object. + * (The caller should make sure srcPath is a file and dstPath is valid) + * + * @param srcKey source key. + * @param dstKey destination key. + * @return true if object is successfully copied. + */ + private boolean singleCopy(String srcKey, String dstKey) { + CopyObjectResult copyResult = + ossClient.copyObject(bucketName, srcKey, bucketName, dstKey); + LOG.debug(copyResult.getETag()); + return true; + } + + /** + * Use multipart copy to copy an OSS object. + * (The caller should make sure srcPath is a file and dstPath is valid) + * + * @param srcKey source key. + * @param contentLength data size of the object to copy. + * @param dstKey destination key. + * @return true if success, or false if upload is aborted. + */ + private boolean multipartCopy(String srcKey, long contentLength, + String dstKey) { + long realPartSize = + AliyunOSSUtils.calculatePartSize(contentLength, uploadPartSize); + int partNum = (int) (contentLength / realPartSize); + if (contentLength % realPartSize != 0) { + partNum++; + } + InitiateMultipartUploadRequest initiateMultipartUploadRequest = + new InitiateMultipartUploadRequest(bucketName, dstKey); + ObjectMetadata meta = new ObjectMetadata(); + if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) { + meta.setServerSideEncryption(serverSideEncryptionAlgorithm); + } + initiateMultipartUploadRequest.setObjectMetadata(meta); + InitiateMultipartUploadResult initiateMultipartUploadResult = + ossClient.initiateMultipartUpload(initiateMultipartUploadRequest); + String uploadId = initiateMultipartUploadResult.getUploadId(); + List partETags = new ArrayList(); + try { + for (int i = 0; i < partNum; i++) { + long skipBytes = realPartSize * i; + long size = (realPartSize < contentLength - skipBytes) ? + realPartSize : contentLength - skipBytes; + UploadPartCopyRequest partCopyRequest = new UploadPartCopyRequest(); + partCopyRequest.setSourceBucketName(bucketName); + partCopyRequest.setSourceKey(srcKey); + partCopyRequest.setBucketName(bucketName); + partCopyRequest.setKey(dstKey); + partCopyRequest.setUploadId(uploadId); + partCopyRequest.setPartSize(size); + partCopyRequest.setBeginIndex(skipBytes); + partCopyRequest.setPartNumber(i + 1); + UploadPartCopyResult partCopyResult = + ossClient.uploadPartCopy(partCopyRequest); + statistics.incrementWriteOps(1); + partETags.add(partCopyResult.getPartETag()); + } + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, dstKey, + uploadId, partETags); + CompleteMultipartUploadResult completeMultipartUploadResult = + ossClient.completeMultipartUpload(completeMultipartUploadRequest); + LOG.debug(completeMultipartUploadResult.getETag()); + return true; + } catch (OSSException | ClientException e) { + AbortMultipartUploadRequest abortMultipartUploadRequest = + new AbortMultipartUploadRequest(bucketName, dstKey, uploadId); + ossClient.abortMultipartUpload(abortMultipartUploadRequest); + return false; + } + } + + /** + * Upload a file as an OSS object, using single upload. + * + * @param key object key. + * @param file local file to upload. + * @throws IOException if failed to upload object. + */ + public void uploadObject(String key, File file) throws IOException { + File object = file.getAbsoluteFile(); + FileInputStream fis = new FileInputStream(object); + ObjectMetadata meta = new ObjectMetadata(); + meta.setContentLength(object.length()); + if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) { + meta.setServerSideEncryption(serverSideEncryptionAlgorithm); + } + try { + PutObjectResult result = ossClient.putObject(bucketName, key, fis, meta); + LOG.debug(result.getETag()); + statistics.incrementWriteOps(1); + } finally { + fis.close(); + } + } + + /** + * Upload a file as an OSS object, using multipart upload. + * + * @param key object key. + * @param file local file to upload. + * @throws IOException if failed to upload object. + */ + public void multipartUploadObject(String key, File file) throws IOException { + File object = file.getAbsoluteFile(); + long dataLen = object.length(); + long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize); + int partNum = (int) (dataLen / realPartSize); + if (dataLen % realPartSize != 0) { + partNum += 1; + } + + InitiateMultipartUploadRequest initiateMultipartUploadRequest = + new InitiateMultipartUploadRequest(bucketName, key); + ObjectMetadata meta = new ObjectMetadata(); + if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) { + meta.setServerSideEncryption(serverSideEncryptionAlgorithm); + } + initiateMultipartUploadRequest.setObjectMetadata(meta); + InitiateMultipartUploadResult initiateMultipartUploadResult = + ossClient.initiateMultipartUpload(initiateMultipartUploadRequest); + List partETags = new ArrayList(); + String uploadId = initiateMultipartUploadResult.getUploadId(); + + try { + for (int i = 0; i < partNum; i++) { + // TODO: Optimize this, avoid opening the object multiple times + FileInputStream fis = new FileInputStream(object); + try { + long skipBytes = realPartSize * i; + AliyunOSSUtils.skipFully(fis, skipBytes); + long size = (realPartSize < dataLen - skipBytes) ? + realPartSize : dataLen - skipBytes; + UploadPartRequest uploadPartRequest = new UploadPartRequest(); + uploadPartRequest.setBucketName(bucketName); + uploadPartRequest.setKey(key); + uploadPartRequest.setUploadId(uploadId); + uploadPartRequest.setInputStream(fis); + uploadPartRequest.setPartSize(size); + uploadPartRequest.setPartNumber(i + 1); + UploadPartResult uploadPartResult = + ossClient.uploadPart(uploadPartRequest); + statistics.incrementWriteOps(1); + partETags.add(uploadPartResult.getPartETag()); + } finally { + fis.close(); + } + } + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, key, + uploadId, partETags); + CompleteMultipartUploadResult completeMultipartUploadResult = + ossClient.completeMultipartUpload(completeMultipartUploadRequest); + LOG.debug(completeMultipartUploadResult.getETag()); + } catch (OSSException | ClientException e) { + AbortMultipartUploadRequest abortMultipartUploadRequest = + new AbortMultipartUploadRequest(bucketName, key, uploadId); + ossClient.abortMultipartUpload(abortMultipartUploadRequest); + } + } + + /** + * list objects. + * + * @param prefix prefix. + * @param maxListingLength max no. of entries + * @param marker last key in any previous search. + * @param recursive whether to list directory recursively. + * @return a list of matches. + */ + public ObjectListing listObjects(String prefix, int maxListingLength, + String marker, boolean recursive) { + String delimiter = recursive ? null : "/"; + prefix = AliyunOSSUtils.maybeAddTrailingSlash(prefix); + ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); + listRequest.setPrefix(prefix); + listRequest.setDelimiter(delimiter); + listRequest.setMaxKeys(maxListingLength); + listRequest.setMarker(marker); + + ObjectListing listing = ossClient.listObjects(listRequest); + statistics.incrementReadOps(1); + return listing; + } + + /** + * Retrieve a part of an object. + * + * @param key the object name that is being retrieved from the Aliyun OSS. + * @param byteStart start position. + * @param byteEnd end position. + * @return This method returns null if the key is not found. + */ + public InputStream retrieve(String key, long byteStart, long byteEnd) { + try { + GetObjectRequest request = new GetObjectRequest(bucketName, key); + request.setRange(byteStart, byteEnd); + return ossClient.getObject(request).getObjectContent(); + } catch (OSSException | ClientException e) { + return null; + } + } + + /** + * Close OSS client properly. + */ + public void close() { + if (ossClient != null) { + ossClient.shutdown(); + ossClient = null; + } + } + + /** + * Clean up all objects matching the prefix. + * + * @param prefix Aliyun OSS object prefix. + */ + public void purge(String prefix) { + String key; + try { + ObjectListing objects = listObjects(prefix, maxKeys, null, true); + for (OSSObjectSummary object : objects.getObjectSummaries()) { + key = object.getKey(); + ossClient.deleteObject(bucketName, key); + } + + for (String dir: objects.getCommonPrefixes()) { + deleteDirs(dir); + } + } catch (OSSException | ClientException e) { + LOG.error("Failed to purge " + prefix); + } + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java new file mode 100644 index 00000000000..b87a3a753bc --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java @@ -0,0 +1,260 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSExceptionMessages; +import org.apache.hadoop.fs.FSInputStream; +import org.apache.hadoop.fs.FileSystem.Statistics; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +/** + * The input stream for OSS blob system. + * The class uses multi-part downloading to read data from the object content + * stream. + */ +public class AliyunOSSInputStream extends FSInputStream { + public static final Log LOG = LogFactory.getLog(AliyunOSSInputStream.class); + private final long downloadPartSize; + private AliyunOSSFileSystemStore store; + private final String key; + private Statistics statistics; + private boolean closed; + private InputStream wrappedStream = null; + private long contentLength; + private long position; + private long partRemaining; + + public AliyunOSSInputStream(Configuration conf, + AliyunOSSFileSystemStore store, String key, Long contentLength, + Statistics statistics) throws IOException { + this.store = store; + this.key = key; + this.statistics = statistics; + this.contentLength = contentLength; + downloadPartSize = conf.getLong(MULTIPART_DOWNLOAD_SIZE_KEY, + MULTIPART_DOWNLOAD_SIZE_DEFAULT); + reopen(0); + closed = false; + } + + /** + * Reopen the wrapped stream at give position, by seeking for + * data of a part length from object content stream. + * + * @param pos position from start of a file + * @throws IOException if failed to reopen + */ + private synchronized void reopen(long pos) throws IOException { + long partSize; + + if (pos < 0) { + throw new EOFException("Cannot seek at negative position:" + pos); + } else if (pos > contentLength) { + throw new EOFException("Cannot seek after EOF, contentLength:" + + contentLength + " position:" + pos); + } else if (pos + downloadPartSize > contentLength) { + partSize = contentLength - pos; + } else { + partSize = downloadPartSize; + } + + if (wrappedStream != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Aborting old stream to open at pos " + pos); + } + wrappedStream.close(); + } + + wrappedStream = store.retrieve(key, pos, pos + partSize -1); + if (wrappedStream == null) { + throw new IOException("Null IO stream"); + } + position = pos; + partRemaining = partSize; + } + + @Override + public synchronized int read() throws IOException { + checkNotClosed(); + + if (partRemaining <= 0 && position < contentLength) { + reopen(position); + } + + int tries = MAX_RETRIES; + boolean retry; + int byteRead = -1; + do { + retry = false; + try { + byteRead = wrappedStream.read(); + } catch (Exception e) { + handleReadException(e, --tries); + retry = true; + } + } while (retry); + if (byteRead >= 0) { + position++; + partRemaining--; + } + + if (statistics != null && byteRead >= 0) { + statistics.incrementBytesRead(1); + } + return byteRead; + } + + + /** + * Verify that the input stream is open. Non blocking; this gives + * the last state of the volatile {@link #closed} field. + * + * @throws IOException if the connection is closed. + */ + private void checkNotClosed() throws IOException { + if (closed) { + throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); + } + } + + @Override + public synchronized int read(byte[] buf, int off, int len) + throws IOException { + checkNotClosed(); + + if (buf == null) { + throw new NullPointerException(); + } else if (off < 0 || len < 0 || len > buf.length - off) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return 0; + } + + int bytesRead = 0; + // Not EOF, and read not done + while (position < contentLength && bytesRead < len) { + if (partRemaining == 0) { + reopen(position); + } + + int tries = MAX_RETRIES; + boolean retry; + int bytes = -1; + do { + retry = false; + try { + bytes = wrappedStream.read(buf, off + bytesRead, len - bytesRead); + } catch (Exception e) { + handleReadException(e, --tries); + retry = true; + } + } while (retry); + + if (bytes > 0) { + bytesRead += bytes; + position += bytes; + partRemaining -= bytes; + } else if (partRemaining != 0) { + throw new IOException("Failed to read from stream. Remaining:" + + partRemaining); + } + } + + if (statistics != null && bytesRead > 0) { + statistics.incrementBytesRead(bytesRead); + } + + // Read nothing, but attempt to read something + if (bytesRead == 0 && len > 0) { + return -1; + } else { + return bytesRead; + } + } + + @Override + public synchronized void close() throws IOException { + if (closed) { + return; + } + closed = true; + if (wrappedStream != null) { + wrappedStream.close(); + } + } + + @Override + public synchronized int available() throws IOException { + checkNotClosed(); + + long remaining = contentLength - position; + if (remaining > Integer.MAX_VALUE) { + return Integer.MAX_VALUE; + } + return (int)remaining; + } + + @Override + public synchronized void seek(long pos) throws IOException { + checkNotClosed(); + if (position == pos) { + return; + } else if (pos > position && pos < position + partRemaining) { + AliyunOSSUtils.skipFully(wrappedStream, pos - position); + position = pos; + } else { + reopen(pos); + } + } + + @Override + public synchronized long getPos() throws IOException { + checkNotClosed(); + return position; + } + + @Override + public boolean seekToNewSource(long targetPos) throws IOException { + checkNotClosed(); + return false; + } + + private void handleReadException(Exception e, int tries) throws IOException{ + if (tries == 0) { + throw new IOException(e); + } + + LOG.warn("Some exceptions occurred in oss connection, try to reopen oss" + + " connection at position '" + position + "', " + e.getMessage()); + try { + Thread.sleep(100); + } catch (InterruptedException e2) { + LOG.warn(e2.getMessage()); + } + reopen(position); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java new file mode 100644 index 00000000000..c75ee187bc9 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem.Statistics; +import org.apache.hadoop.fs.LocalDirAllocator; +import org.apache.hadoop.util.Progressable; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +/** + * The output stream for OSS blob system. + * Data will be buffered on local disk, then uploaded to OSS in + * {@link #close()} method. + */ +public class AliyunOSSOutputStream extends OutputStream { + public static final Log LOG = LogFactory.getLog(AliyunOSSOutputStream.class); + private AliyunOSSFileSystemStore store; + private final String key; + private Statistics statistics; + private Progressable progress; + private long partSizeThreshold; + private LocalDirAllocator dirAlloc; + private boolean closed; + private File tmpFile; + private BufferedOutputStream backupStream; + + public AliyunOSSOutputStream(Configuration conf, + AliyunOSSFileSystemStore store, String key, Progressable progress, + Statistics statistics) throws IOException { + this.store = store; + this.key = key; + // The caller cann't get any progress information + this.progress = progress; + this.statistics = statistics; + partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, + MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT); + + if (conf.get(BUFFER_DIR_KEY) == null) { + conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss"); + } + dirAlloc = new LocalDirAllocator(BUFFER_DIR_KEY); + + tmpFile = dirAlloc.createTmpFileForWrite("output-", + LocalDirAllocator.SIZE_UNKNOWN, conf); + backupStream = new BufferedOutputStream(new FileOutputStream(tmpFile)); + closed = false; + } + + @Override + public synchronized void close() throws IOException { + if (closed) { + return; + } + closed = true; + if (backupStream != null) { + backupStream.close(); + } + long dataLen = tmpFile.length(); + try { + if (dataLen <= partSizeThreshold) { + store.uploadObject(key, tmpFile); + } else { + store.multipartUploadObject(key, tmpFile); + } + } finally { + if (!tmpFile.delete()) { + LOG.warn("Can not delete file: " + tmpFile); + } + } + } + + + + @Override + public synchronized void flush() throws IOException { + backupStream.flush(); + } + + @Override + public synchronized void write(int b) throws IOException { + backupStream.write(b); + statistics.incrementBytesWritten(1); + } + +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java new file mode 100644 index 00000000000..cae9749a614 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import java.io.IOException; +import java.io.InputStream; + +import com.aliyun.oss.common.auth.CredentialsProvider; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.ProviderUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +/** + * Utility methods for Aliyun OSS code. + */ +final public class AliyunOSSUtils { + private static final Logger LOG = + LoggerFactory.getLogger(AliyunOSSUtils.class); + + private AliyunOSSUtils() { + } + + /** + * Used to get password from configuration. + * + * @param conf configuration that contains password information + * @param key the key of the password + * @return the value for the key + * @throws IOException if failed to get password from configuration + */ + public static String getValueWithKey(Configuration conf, String key) + throws IOException { + try { + final char[] pass = conf.getPassword(key); + if (pass != null) { + return (new String(pass)).trim(); + } else { + return ""; + } + } catch (IOException ioe) { + throw new IOException("Cannot find password option " + key, ioe); + } + } + + /** + * Skip the requested number of bytes or fail if there are no enough bytes + * left. This allows for the possibility that {@link InputStream#skip(long)} + * may not skip as many bytes as requested (most likely because of reaching + * EOF). + * + * @param is the input stream to skip. + * @param n the number of bytes to skip. + * @throws IOException thrown when skipped less number of bytes. + */ + public static void skipFully(InputStream is, long n) throws IOException { + long total = 0; + long cur = 0; + + do { + cur = is.skip(n - total); + total += cur; + } while((total < n) && (cur > 0)); + + if (total < n) { + throw new IOException("Failed to skip " + n + " bytes, possibly due " + + "to EOF."); + } + } + + /** + * Calculate a proper size of multipart piece. If minPartSize + * is too small, the number of multipart pieces may exceed the limit of + * {@link Constants#MULTIPART_UPLOAD_PART_NUM_LIMIT}. + * + * @param contentLength the size of file. + * @param minPartSize the minimum size of multipart piece. + * @return a revisional size of multipart piece. + */ + public static long calculatePartSize(long contentLength, long minPartSize) { + long tmpPartSize = contentLength / MULTIPART_UPLOAD_PART_NUM_LIMIT + 1; + return Math.max(minPartSize, tmpPartSize); + } + + /** + * Create credential provider specified by configuration, or create default + * credential provider if not specified. + * + * @param conf configuration + * @return a credential provider + * @throws IOException on any problem. Class construction issues may be + * nested inside the IOE. + */ + public static CredentialsProvider getCredentialsProvider(Configuration conf) + throws IOException { + CredentialsProvider credentials; + + String className = conf.getTrimmed(ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY); + if (StringUtils.isEmpty(className)) { + Configuration newConf = + ProviderUtils.excludeIncompatibleCredentialProviders(conf, + AliyunOSSFileSystem.class); + credentials = new AliyunCredentialsProvider(newConf); + } else { + try { + LOG.debug("Credential provider class is:" + className); + Class credClass = Class.forName(className); + try { + credentials = + (CredentialsProvider)credClass.getDeclaredConstructor( + Configuration.class).newInstance(conf); + } catch (NoSuchMethodException | SecurityException e) { + credentials = + (CredentialsProvider)credClass.getDeclaredConstructor() + .newInstance(); + } + } catch (ClassNotFoundException e) { + throw new IOException(className + " not found.", e); + } catch (NoSuchMethodException | SecurityException e) { + throw new IOException(String.format("%s constructor exception. A " + + "class specified in %s must provide an accessible constructor " + + "accepting URI and Configuration, or an accessible default " + + "constructor.", className, ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY), + e); + } catch (ReflectiveOperationException | IllegalArgumentException e) { + throw new IOException(className + " instantiation exception.", e); + } + } + + return credentials; + } + + /** + * Turns a path (relative or otherwise) into an OSS key, adding a trailing + * "/" if the path is not the root and does not already have a "/" + * at the end. + * + * @param key OSS key or "" + * @return the with a trailing "/", or, if it is the root key, "". + */ + public static String maybeAddTrailingSlash(String key) { + if (StringUtils.isNotEmpty(key) && !key.endsWith("/")) { + return key + '/'; + } else { + return key; + } + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java new file mode 100644 index 00000000000..04a2ccd6c54 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +/** + * ALL configuration constants for OSS filesystem. + */ +public final class Constants { + + private Constants() { + } + + // Class of credential provider + public static final String ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY = + "fs.oss.credentials.provider"; + + // OSS access verification + public static final String ACCESS_KEY_ID = "fs.oss.accessKeyId"; + public static final String ACCESS_KEY_SECRET = "fs.oss.accessKeySecret"; + public static final String SECURITY_TOKEN = "fs.oss.securityToken"; + + // Number of simultaneous connections to oss + public static final String MAXIMUM_CONNECTIONS_KEY = + "fs.oss.connection.maximum"; + public static final int MAXIMUM_CONNECTIONS_DEFAULT = 32; + + // Connect to oss over ssl + public static final String SECURE_CONNECTIONS_KEY = + "fs.oss.connection.secure.enabled"; + public static final boolean SECURE_CONNECTIONS_DEFAULT = true; + + // Use a custom endpoint + public static final String ENDPOINT_KEY = "fs.oss.endpoint"; + + // Connect to oss through a proxy server + public static final String PROXY_HOST_KEY = "fs.oss.proxy.host"; + public static final String PROXY_PORT_KEY = "fs.oss.proxy.port"; + public static final String PROXY_USERNAME_KEY = "fs.oss.proxy.username"; + public static final String PROXY_PASSWORD_KEY = "fs.oss.proxy.password"; + public static final String PROXY_DOMAIN_KEY = "fs.oss.proxy.domain"; + public static final String PROXY_WORKSTATION_KEY = + "fs.oss.proxy.workstation"; + + // Number of times we should retry errors + public static final String MAX_ERROR_RETRIES_KEY = "fs.oss.attempts.maximum"; + public static final int MAX_ERROR_RETRIES_DEFAULT = 20; + + // Time until we give up trying to establish a connection to oss + public static final String ESTABLISH_TIMEOUT_KEY = + "fs.oss.connection.establish.timeout"; + public static final int ESTABLISH_TIMEOUT_DEFAULT = 50000; + + // Time until we give up on a connection to oss + public static final String SOCKET_TIMEOUT_KEY = "fs.oss.connection.timeout"; + public static final int SOCKET_TIMEOUT_DEFAULT = 200000; + + // Number of records to get while paging through a directory listing + public static final String MAX_PAGING_KEYS_KEY = "fs.oss.paging.maximum"; + public static final int MAX_PAGING_KEYS_DEFAULT = 1000; + + // Size of each of or multipart pieces in bytes + public static final String MULTIPART_UPLOAD_SIZE_KEY = + "fs.oss.multipart.upload.size"; + + public static final long MULTIPART_UPLOAD_SIZE_DEFAULT = 10 * 1024 * 1024; + public static final int MULTIPART_UPLOAD_PART_NUM_LIMIT = 10000; + + // Minimum size in bytes before we start a multipart uploads or copy + public static final String MIN_MULTIPART_UPLOAD_THRESHOLD_KEY = + "fs.oss.multipart.upload.threshold"; + public static final long MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT = + 20 * 1024 * 1024; + + public static final String MULTIPART_DOWNLOAD_SIZE_KEY = + "fs.oss.multipart.download.size"; + + public static final long MULTIPART_DOWNLOAD_SIZE_DEFAULT = 100 * 1024; + + // Comma separated list of directories + public static final String BUFFER_DIR_KEY = "fs.oss.buffer.dir"; + + // private | public-read | public-read-write + public static final String CANNED_ACL_KEY = "fs.oss.acl.default"; + public static final String CANNED_ACL_DEFAULT = ""; + + // OSS server-side encryption + public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY = + "fs.oss.server-side-encryption-algorithm"; + + public static final String FS_OSS_BLOCK_SIZE_KEY = "fs.oss.block.size"; + public static final int FS_OSS_BLOCK_SIZE_DEFAULT = 64 * 1024 * 1024; + public static final String FS_OSS = "oss"; + + public static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 100 * 1024L; + public static final int MAX_RETRIES = 10; + +} diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java new file mode 100644 index 00000000000..234567b2b00 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Aliyun OSS Filesystem. + */ +package org.apache.hadoop.fs.aliyun.oss; \ No newline at end of file diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md new file mode 100644 index 00000000000..88c83b5724f --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md @@ -0,0 +1,294 @@ + + +# Hadoop-Aliyun module: Integration with Aliyun Web Services + + + +## Overview + +The `hadoop-aliyun` module provides support for Aliyun integration with +[Aliyun Object Storage Service (Aliyun OSS)](https://www.aliyun.com/product/oss). +The generated JAR file, `hadoop-aliyun.jar` also declares a transitive +dependency on all external artifacts which are needed for this support — enabling +downstream applications to easily use this support. + +To make it part of Apache Hadoop's default classpath, simply make sure +that HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-aliyun' in the list. + +### Features + +* Read and write data stored in Aliyun OSS. +* Present a hierarchical file system view by implementing the standard Hadoop +[`FileSystem`](../api/org/apache/hadoop/fs/FileSystem.html) interface. +* Can act as a source of data in a MapReduce job, or a sink. + +### Warning #1: Object Stores are not filesystems. + +Aliyun OSS is an example of "an object store". In order to achieve scalability +and especially high availability, Aliyun OSS has relaxed some of the constraints +which classic "POSIX" filesystems promise. + + + +Specifically + +1. Atomic operations: `delete()` and `rename()` are implemented by recursive +file-by-file operations. They take time at least proportional to the number of files, +during which time partial updates may be visible. `delete()` and `rename()` +can not guarantee atomicity. If the operations are interrupted, the filesystem +is left in an intermediate state. +2. File owner and group are persisted, but the permissions model is not enforced. +Authorization occurs at the level of the entire Aliyun account via +[Aliyun Resource Access Management (Aliyun RAM)](https://www.aliyun.com/product/ram). +3. Directory last access time is not tracked. +4. The append operation is not supported. + +### Warning #2: Directory last access time is not tracked, +features of Hadoop relying on this can have unexpected behaviour. E.g. the +AggregatedLogDeletionService of YARN will not remove the appropriate logfiles. + +### Warning #3: Your Aliyun credentials are valuable + +Your Aliyun credentials not only pay for services, they offer read and write +access to the data. Anyone with the account can not only read your datasets +—they can delete them. + +Do not inadvertently share these credentials through means such as +1. Checking in to SCM any configuration files containing the secrets. +2. Logging them to a console, as they invariably end up being seen. +3. Defining filesystem URIs with the credentials in the URL, such as +`oss://accessKeyId:accessKeySecret@directory/file`. They will end up in +logs and error messages. +4. Including the secrets in bug reports. + +If you do any of these: change your credentials immediately! + +### Warning #4: The Aliyun OSS client provided by Aliyun E-MapReduce are different from this implementation + +Specifically: on Aliyun E-MapReduce, `oss://` is also supported but with +a different implementation. If you are using Aliyun E-MapReduce, +follow these instructions —and be aware that all issues related to Aliyun +OSS integration in E-MapReduce can only be addressed by Aliyun themselves: +please raise your issues with them. + +## OSS + +### Authentication properties + + + fs.oss.accessKeyId + Aliyun access key ID + + + + fs.oss.accessKeySecret + Aliyun access key secret + + + + fs.oss.credentials.provider + + Class name of a credentials provider that implements + com.aliyun.oss.common.auth.CredentialsProvider. Omit if using access/secret keys + or another authentication mechanism. The specified class must provide an + accessible constructor accepting java.net.URI and + org.apache.hadoop.conf.Configuration, or an accessible default constructor. + + + +### Other properties + + + fs.oss.endpoint + Aliyun OSS endpoint to connect to. An up-to-date list is + provided in the Aliyun OSS Documentation. + + + + + fs.oss.proxy.host + Hostname of the (optinal) proxy server for Aliyun OSS connection + + + + fs.oss.proxy.port + Proxy server port + + + + fs.oss.proxy.username + Username for authenticating with proxy server + + + + fs.oss.proxy.password + Password for authenticating with proxy server. + + + + fs.oss.proxy.domain + Domain for authenticating with proxy server. + + + + fs.oss.proxy.workstation + Workstation for authenticating with proxy server. + + + + fs.oss.attempts.maximum + 20 + How many times we should retry commands on transient errors. + + + + fs.oss.connection.establish.timeout + 50000 + Connection setup timeout in milliseconds. + + + + fs.oss.connection.timeout + 200000 + Socket connection timeout in milliseconds. + + + + fs.oss.paging.maximum + 1000 + How many keys to request from Aliyun OSS when doing directory listings at a time. + + + + + fs.oss.multipart.upload.size + 10485760 + Size of each of multipart pieces in bytes. + + + + fs.oss.multipart.upload.threshold + 20971520 + Minimum size in bytes before we start a multipart uploads or copy. + + + + fs.oss.multipart.download.size + 102400/value> + Size in bytes in each request from ALiyun OSS. + + + + fs.oss.buffer.dir + Comma separated list of directories to buffer OSS data before uploading to Aliyun OSS + + + + fs.oss.acl.default + + Set a canned ACL for bucket. Value may be private, public-read, public-read-write. + + + + + fs.oss.server-side-encryption-algorithm + + Specify a server-side encryption algorithm for oss: file system. + Unset by default, and the only other currently allowable value is AES256. + + + + + fs.oss.connection.maximum + 32 + Number of simultaneous connections to oss. + + + + fs.oss.connection.secure.enabled + true + Connect to oss over ssl or not, true by default. + + +## Testing the hadoop-aliyun Module + +To test `oss://` filesystem client, two files which pass in authentication +details to the test runner are needed. + +1. `auth-keys.xml` +2. `core-site.xml` + +Those two configuration files must be put into +`hadoop-tools/hadoop-aliyun/src/test/resources`. + +### `core-site.xml` + +This file pre-exists and sources the configurations created in `auth-keys.xml`. + +For most cases, no modification is needed, unless a specific, non-default property +needs to be set during the testing. + +### `auth-keys.xml` + +This file triggers the testing of Aliyun OSS module. Without this file, +*none of the tests in this module will be executed* + +It contains the access key Id/secret and proxy information that are needed to +connect to Aliyun OSS, and an OSS bucket URL should be also provided. + +1. `test.fs.oss.name` : the URL of the bucket for Aliyun OSS tests + +The contents of the bucket will be cleaned during the testing process, so +do not use the bucket for any purpose other than testing. + +### Run Hadoop contract tests +Create file `contract-test-options.xml` under `/test/resources`. If a +specific file `fs.contract.test.fs.oss` test path is not defined, those +tests will be skipped. Credentials are also needed to run any of those +tests, they can be copied from `auth-keys.xml` or through direct +XInclude inclusion. Here is an example of `contract-test-options.xml`: + + + + + + + + + fs.contract.test.fs.oss + oss://spark-tests + + + + fs.oss.impl + org.apache.hadoop.fs.aliyun.AliyunOSSFileSystem + + + + fs.oss.endpoint + oss-cn-hangzhou.aliyuncs.com + + + + fs.oss.buffer.dir + /tmp/oss + + + + fs.oss.multipart.download.size + 102400 + + diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java new file mode 100644 index 00000000000..901cb2bd082 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.junit.internal.AssumptionViolatedException; + +import java.io.IOException; +import java.net.URI; + +/** + * Utility class for Aliyun OSS Tests. + */ +public final class AliyunOSSTestUtils { + + private AliyunOSSTestUtils() { + } + + /** + * Create the test filesystem. + * + * If the test.fs.oss.name property is not set, + * tests will fail. + * + * @param conf configuration + * @return the FS + * @throws IOException + */ + public static AliyunOSSFileSystem createTestFileSystem(Configuration conf) + throws IOException { + String fsname = conf.getTrimmed( + TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME, ""); + + boolean liveTest = StringUtils.isNotEmpty(fsname); + URI testURI = null; + if (liveTest) { + testURI = URI.create(fsname); + liveTest = testURI.getScheme().equals(Constants.FS_OSS); + } + + if (!liveTest) { + throw new AssumptionViolatedException("No test filesystem in " + + TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME); + } + AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem(); + ossfs.initialize(testURI, conf); + return ossfs; + } + + /** + * Generate unique test path for multiple user tests. + * + * @return root test path + */ + public static String generateUniqueTestPath() { + String testUniqueForkId = System.getProperty("test.unique.fork.id"); + return testUniqueForkId == null ? "/test" : + "/" + testUniqueForkId + "/test"; + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java new file mode 100644 index 00000000000..e08a4dccfca --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import com.aliyun.oss.common.auth.Credentials; +import com.aliyun.oss.common.auth.InvalidCredentialsException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; +import org.junit.Test; + +import java.io.IOException; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID; +import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET; +import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN; + +/** + * Tests use of temporary credentials (for example, Aliyun STS & Aliyun OSS). + * This test extends a class that "does things to the root directory", and + * should only be used against transient filesystems where you don't care about + * the data. + */ +public class TestAliyunCredentials extends AbstractFSContractTestBase { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); + } + + @Test + public void testCredentialMissingAccessKeyId() throws Throwable { + Configuration conf = new Configuration(); + conf.set(ACCESS_KEY_ID, ""); + conf.set(ACCESS_KEY_SECRET, "accessKeySecret"); + conf.set(SECURITY_TOKEN, "token"); + validateCredential(conf); + } + + @Test + public void testCredentialMissingAccessKeySecret() throws Throwable { + Configuration conf = new Configuration(); + conf.set(ACCESS_KEY_ID, "accessKeyId"); + conf.set(ACCESS_KEY_SECRET, ""); + conf.set(SECURITY_TOKEN, "token"); + validateCredential(conf); + } + + private void validateCredential(Configuration conf) { + try { + AliyunCredentialsProvider provider + = new AliyunCredentialsProvider(conf); + Credentials credentials = provider.getCredentials(); + fail("Expected a CredentialInitializationException, got " + credentials); + } catch (InvalidCredentialsException expected) { + // expected + } catch (IOException e) { + fail("Unexpected exception."); + } + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java new file mode 100644 index 00000000000..ad8ef6ee3a4 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileSystemContractBaseTest; +import org.apache.hadoop.fs.Path; + +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * Tests a live Aliyun OSS system. + * + * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from + * TestCase which uses the old Junit3 runner that doesn't ignore assumptions + * properly making it impossible to skip the tests if we don't have a valid + * bucket. + */ +public class TestAliyunOSSFileSystemContract + extends FileSystemContractBaseTest { + public static final String TEST_FS_OSS_NAME = "test.fs.oss.name"; + private static String testRootPath = + AliyunOSSTestUtils.generateUniqueTestPath(); + + @Override + public void setUp() throws Exception { + Configuration conf = new Configuration(); + fs = AliyunOSSTestUtils.createTestFileSystem(conf); + super.setUp(); + } + + @Override + public void tearDown() throws Exception { + if (fs != null) { + fs.delete(super.path(testRootPath), true); + } + super.tearDown(); + } + + @Override + protected Path path(String path) { + if (path.startsWith("/")) { + return super.path(testRootPath + path); + } else { + return super.path(testRootPath + "/" + path); + } + } + + @Override + public void testMkdirsWithUmask() throws Exception { + // not supported + } + + @Override + public void testRootDirAlwaysExists() throws Exception { + //this will throw an exception if the path is not found + fs.getFileStatus(super.path("/")); + //this catches overrides of the base exists() method that don't + //use getFileStatus() as an existence probe + assertTrue("FileSystem.exists() fails for root", + fs.exists(super.path("/"))); + } + + @Override + public void testRenameRootDirForbidden() throws Exception { + if (!renameSupported()) { + return; + } + rename(super.path("/"), + super.path("/test/newRootDir"), + false, true, false); + } + + public void testDeleteSubdir() throws IOException { + Path parentDir = this.path("/test/hadoop"); + Path file = this.path("/test/hadoop/file"); + Path subdir = this.path("/test/hadoop/subdir"); + this.createFile(file); + + assertTrue("Created subdir", this.fs.mkdirs(subdir)); + assertTrue("File exists", this.fs.exists(file)); + assertTrue("Parent dir exists", this.fs.exists(parentDir)); + assertTrue("Subdir exists", this.fs.exists(subdir)); + + assertTrue("Deleted subdir", this.fs.delete(subdir, true)); + assertTrue("Parent should exist", this.fs.exists(parentDir)); + + assertTrue("Deleted file", this.fs.delete(file, false)); + assertTrue("Parent should exist", this.fs.exists(parentDir)); + } + + + @Override + protected boolean renameSupported() { + return true; + } + + @Override + public void testRenameNonExistentPath() throws Exception { + if (this.renameSupported()) { + Path src = this.path("/test/hadoop/path"); + Path dst = this.path("/test/new/newpath"); + try { + super.rename(src, dst, false, false, false); + fail("Should throw FileNotFoundException!"); + } catch (FileNotFoundException e) { + // expected + } + } + } + + @Override + public void testRenameFileMoveToNonExistentDirectory() throws Exception { + if (this.renameSupported()) { + Path src = this.path("/test/hadoop/file"); + this.createFile(src); + Path dst = this.path("/test/new/newfile"); + try { + super.rename(src, dst, false, true, false); + fail("Should throw FileNotFoundException!"); + } catch (FileNotFoundException e) { + // expected + } + } + } + + @Override + public void testRenameDirectoryMoveToNonExistentDirectory() throws Exception { + if (this.renameSupported()) { + Path src = this.path("/test/hadoop/dir"); + this.fs.mkdirs(src); + Path dst = this.path("/test/new/newdir"); + try { + super.rename(src, dst, false, true, false); + fail("Should throw FileNotFoundException!"); + } catch (FileNotFoundException e) { + // expected + } + } + } + + @Override + public void testRenameFileMoveToExistingDirectory() throws Exception { + super.testRenameFileMoveToExistingDirectory(); + } + + @Override + public void testRenameFileAsExistingFile() throws Exception { + if (this.renameSupported()) { + Path src = this.path("/test/hadoop/file"); + this.createFile(src); + Path dst = this.path("/test/new/newfile"); + this.createFile(dst); + try { + super.rename(src, dst, false, true, true); + fail("Should throw FileAlreadyExistsException"); + } catch (FileAlreadyExistsException e) { + // expected + } + } + } + + @Override + public void testRenameDirectoryAsExistingFile() throws Exception { + if (this.renameSupported()) { + Path src = this.path("/test/hadoop/dir"); + this.fs.mkdirs(src); + Path dst = this.path("/test/new/newfile"); + this.createFile(dst); + try { + super.rename(src, dst, false, true, true); + fail("Should throw FileAlreadyExistsException"); + } catch (FileAlreadyExistsException e) { + // expected + } + } + } + + public void testGetFileStatusFileAndDirectory() throws Exception { + Path filePath = this.path("/test/oss/file1"); + this.createFile(filePath); + assertTrue("Should be file", this.fs.getFileStatus(filePath).isFile()); + assertFalse("Should not be directory", + this.fs.getFileStatus(filePath).isDirectory()); + + Path dirPath = this.path("/test/oss/dir"); + this.fs.mkdirs(dirPath); + assertTrue("Should be directory", + this.fs.getFileStatus(dirPath).isDirectory()); + assertFalse("Should not be file", this.fs.getFileStatus(dirPath).isFile()); + } + + public void testMkdirsForExistingFile() throws Exception { + Path testFile = this.path("/test/hadoop/file"); + assertFalse(this.fs.exists(testFile)); + this.createFile(testFile); + assertTrue(this.fs.exists(testFile)); + try { + this.fs.mkdirs(testFile); + fail("Should throw FileAlreadyExistsException!"); + } catch (FileAlreadyExistsException e) { + // expected + } + } + + public void testWorkingDirectory() throws Exception { + Path workDir = super.path(this.getDefaultWorkingDirectory()); + assertEquals(workDir, this.fs.getWorkingDirectory()); + this.fs.setWorkingDirectory(super.path(".")); + assertEquals(workDir, this.fs.getWorkingDirectory()); + this.fs.setWorkingDirectory(super.path("..")); + assertEquals(workDir.getParent(), this.fs.getWorkingDirectory()); + Path relativeDir = super.path("hadoop"); + this.fs.setWorkingDirectory(relativeDir); + assertEquals(relativeDir, this.fs.getWorkingDirectory()); + Path absoluteDir = super.path("/test/hadoop"); + this.fs.setWorkingDirectory(absoluteDir); + assertEquals(absoluteDir, this.fs.getWorkingDirectory()); + } + +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java new file mode 100644 index 00000000000..7f4bac25642 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.security.DigestInputStream; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeNotNull; + +/** + * Test the bridging logic between Hadoop's abstract filesystem and + * Aliyun OSS. + */ +public class TestAliyunOSSFileSystemStore { + private Configuration conf; + private AliyunOSSFileSystemStore store; + private AliyunOSSFileSystem fs; + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + fs = new AliyunOSSFileSystem(); + fs.initialize(URI.create(conf.get("test.fs.oss.name")), conf); + store = fs.getStore(); + } + + @After + public void tearDown() throws Exception { + try { + store.purge("test"); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + } + + @BeforeClass + public static void checkSettings() throws Exception { + Configuration conf = new Configuration(); + assumeNotNull(conf.get(Constants.ACCESS_KEY_ID)); + assumeNotNull(conf.get(Constants.ACCESS_KEY_SECRET)); + assumeNotNull(conf.get("test.fs.oss.name")); + } + + protected void writeRenameReadCompare(Path path, long len) + throws IOException, NoSuchAlgorithmException { + // If len > fs.oss.multipart.upload.threshold, + // we'll use a multipart upload copy + MessageDigest digest = MessageDigest.getInstance("MD5"); + OutputStream out = new BufferedOutputStream( + new DigestOutputStream(fs.create(path, false), digest)); + for (long i = 0; i < len; i++) { + out.write('Q'); + } + out.flush(); + out.close(); + + assertTrue("Exists", fs.exists(path)); + + Path copyPath = path.suffix(".copy"); + fs.rename(path, copyPath); + + assertTrue("Copy exists", fs.exists(copyPath)); + + // Download file from Aliyun OSS and compare the digest against the original + MessageDigest digest2 = MessageDigest.getInstance("MD5"); + InputStream in = new BufferedInputStream( + new DigestInputStream(fs.open(copyPath), digest2)); + long copyLen = 0; + while (in.read() != -1) { + copyLen++; + } + in.close(); + + assertEquals("Copy length matches original", len, copyLen); + assertArrayEquals("Digests match", digest.digest(), digest2.digest()); + } + + @Test + public void testSmallUpload() throws IOException, NoSuchAlgorithmException { + // Regular upload, regular copy + writeRenameReadCompare(new Path("/test/small"), 16384); + } + + @Test + public void testLargeUpload() + throws IOException, NoSuchAlgorithmException { + // Multipart upload, multipart copy + writeRenameReadCompare(new Path("/test/xlarge"), 52428800L); // 50MB byte + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java new file mode 100644 index 00000000000..37af28ff06a --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.io.IOUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Random; + +import static org.junit.Assert.assertTrue; + +/** + * Tests basic functionality for AliyunOSSInputStream, including seeking and + * reading files. + */ +public class TestAliyunOSSInputStream { + + private FileSystem fs; + + private static final Logger LOG = + LoggerFactory.getLogger(TestAliyunOSSInputStream.class); + + private static String testRootPath = + AliyunOSSTestUtils.generateUniqueTestPath(); + + @Rule + public Timeout testTimeout = new Timeout(30 * 60 * 1000); + + @Before + public void setUp() throws Exception { + Configuration conf = new Configuration(); + fs = AliyunOSSTestUtils.createTestFileSystem(conf); + } + + @After + public void tearDown() throws Exception { + if (fs != null) { + fs.delete(new Path(testRootPath), true); + } + } + + private Path setPath(String path) { + if (path.startsWith("/")) { + return new Path(testRootPath + path); + } else { + return new Path(testRootPath + "/" + path); + } + } + + @Test + public void testSeekFile() throws Exception { + Path smallSeekFile = setPath("/test/smallSeekFile.txt"); + long size = 5 * 1024 * 1024; + + ContractTestUtils.generateTestFile(this.fs, smallSeekFile, size, 256, 255); + LOG.info("5MB file created: smallSeekFile.txt"); + + FSDataInputStream instream = this.fs.open(smallSeekFile); + int seekTimes = 5; + LOG.info("multiple fold position seeking test...:"); + for (int i = 0; i < seekTimes; i++) { + long pos = size / (seekTimes - i) - 1; + LOG.info("begin seeking for pos: " + pos); + instream.seek(pos); + assertTrue("expected position at:" + pos + ", but got:" + + instream.getPos(), instream.getPos() == pos); + LOG.info("completed seeking at pos: " + instream.getPos()); + } + LOG.info("random position seeking test...:"); + Random rand = new Random(); + for (int i = 0; i < seekTimes; i++) { + long pos = Math.abs(rand.nextLong()) % size; + LOG.info("begin seeking for pos: " + pos); + instream.seek(pos); + assertTrue("expected position at:" + pos + ", but got:" + + instream.getPos(), instream.getPos() == pos); + LOG.info("completed seeking at pos: " + instream.getPos()); + } + IOUtils.closeStream(instream); + } + + @Test + public void testReadFile() throws Exception { + final int bufLen = 256; + final int sizeFlag = 5; + String filename = "readTestFile_" + sizeFlag + ".txt"; + Path readTestFile = setPath("/test/" + filename); + long size = sizeFlag * 1024 * 1024; + + ContractTestUtils.generateTestFile(this.fs, readTestFile, size, 256, 255); + LOG.info(sizeFlag + "MB file created: /test/" + filename); + + FSDataInputStream instream = this.fs.open(readTestFile); + byte[] buf = new byte[bufLen]; + long bytesRead = 0; + while (bytesRead < size) { + int bytes; + if (size - bytesRead < bufLen) { + int remaining = (int)(size - bytesRead); + bytes = instream.read(buf, 0, remaining); + } else { + bytes = instream.read(buf, 0, bufLen); + } + bytesRead += bytes; + + if (bytesRead % (1024 * 1024) == 0) { + int available = instream.available(); + int remaining = (int)(size - bytesRead); + assertTrue("expected remaining:" + remaining + ", but got:" + available, + remaining == available); + LOG.info("Bytes read: " + Math.round((double)bytesRead / (1024 * 1024)) + + " MB"); + } + } + assertTrue(instream.available() == 0); + IOUtils.closeStream(instream); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java new file mode 100644 index 00000000000..6b87d9ca466 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.IOException; + +/** + * Tests regular and multi-part upload functionality for AliyunOSSOutputStream. + */ +public class TestAliyunOSSOutputStream { + private FileSystem fs; + private static String testRootPath = + AliyunOSSTestUtils.generateUniqueTestPath(); + + @Rule + public Timeout testTimeout = new Timeout(30 * 60 * 1000); + + @Before + public void setUp() throws Exception { + Configuration conf = new Configuration(); + conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024); + conf.setInt(Constants.MULTIPART_UPLOAD_SIZE_KEY, 5 * 1024 * 1024); + fs = AliyunOSSTestUtils.createTestFileSystem(conf); + } + + @After + public void tearDown() throws Exception { + if (fs != null) { + fs.delete(new Path(testRootPath), true); + } + } + + protected Path getTestPath() { + return new Path(testRootPath + "/test-aliyun-oss"); + } + + @Test + public void testRegularUpload() throws IOException { + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024); + } + + @Test + public void testMultiPartUpload() throws IOException { + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024); + } + + @Test + public void testMultiPartUploadLimit() throws IOException { + long partSize1 = AliyunOSSUtils.calculatePartSize(10 * 1024, 100 * 1024); + assert(10 * 1024 / partSize1 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT); + + long partSize2 = AliyunOSSUtils.calculatePartSize(200 * 1024, 100 * 1024); + assert(200 * 1024 / partSize2 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT); + + long partSize3 = AliyunOSSUtils.calculatePartSize(10000 * 100 * 1024, + 100 * 1024); + assert(10000 * 100 * 1024 / partSize3 + < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT); + + long partSize4 = AliyunOSSUtils.calculatePartSize(10001 * 100 * 1024, + 100 * 1024); + assert(10001 * 100 * 1024 / partSize4 + < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java new file mode 100644 index 00000000000..624c606c6b8 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.AbstractBondedFSContract; + +/** + * The contract of Aliyun OSS: only enabled if the test bucket is provided. + */ +public class AliyunOSSContract extends AbstractBondedFSContract { + + public static final String CONTRACT_XML = "contract/aliyun-oss.xml"; + + public AliyunOSSContract(Configuration conf) { + super(conf); + //insert the base features + addConfResource(CONTRACT_XML); + } + + @Override + public String getScheme() { + return "oss"; + } + + @Override + public Path getTestPath() { + String testUniqueForkId = System.getProperty("test.unique.fork.id"); + return testUniqueForkId == null ? super.getTestPath() : + new Path("/" + testUniqueForkId, "test"); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java similarity index 53% rename from hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java index 7a7af06f04c..88dd8cd2267 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java @@ -15,26 +15,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdfs.server.namenode; -import java.io.IOException; -import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.OpInstanceCache; +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; /** - * Utilities for testing edit logs + * Aliyun OSS contract creating tests. */ -public class FSEditLogTestUtil { - private static OpInstanceCache cache = new OpInstanceCache(); +public class TestAliyunOSSContractCreate extends AbstractContractCreateTest { - public static FSEditLogOp getNoOpInstance() { - return FSEditLogOp.LogSegmentOp.getInstance(cache, - FSEditLogOpCodes.OP_END_LOG_SEGMENT); + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); } - public static long countTransactionsInStream(EditLogInputStream in) - throws IOException { - FSEditLogLoader.EditLogValidation validation = - FSEditLogLoader.scanEditLog(in, Long.MAX_VALUE); - return (validation.getEndTxId() - in.getFirstTxId()) + 1; - } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java new file mode 100644 index 00000000000..1658d806831 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Aliyun OSS contract deleting tests. + */ +public class TestAliyunOSSContractDelete extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java new file mode 100644 index 00000000000..18d09d54149 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; + +import static org.apache.hadoop.fs.aliyun.oss.Constants.*; + +/** + * Contract test suite covering Aliyun OSS integration with DistCp. + */ +public class TestAliyunOSSContractDistCp extends AbstractContractDistCpTest { + + private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB + + @Override + protected Configuration createConfiguration() { + Configuration newConf = super.createConfiguration(); + newConf.setLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, MULTIPART_SETTING); + newConf.setLong(MULTIPART_UPLOAD_SIZE_KEY, MULTIPART_SETTING); + return newConf; + } + + @Override + protected AliyunOSSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java new file mode 100644 index 00000000000..c69124d9243 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Test getFileStatus and related listing operations. + */ +public class TestAliyunOSSContractGetFileStatus + extends AbstractContractGetFileStatusTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); + } + +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java new file mode 100644 index 00000000000..6cb754975ca --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Aliyun OSS contract directory tests. + */ +public class TestAliyunOSSContractMkdir extends AbstractContractMkdirTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java new file mode 100644 index 00000000000..099aba6296f --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Aliyun OSS contract opening file tests. + */ +public class TestAliyunOSSContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/proto/bkjournal.proto b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java similarity index 52% rename from hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/proto/bkjournal.proto rename to hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java index 15fa479ea59..e15b3ba30de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/proto/bkjournal.proto +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java @@ -16,34 +16,20 @@ * limitations under the License. */ -// This file contains protocol buffers that are used by bkjournal -// mostly for storing data in zookeeper +package org.apache.hadoop.fs.aliyun.oss.contract; -option java_package = "org.apache.hadoop.contrib.bkjournal"; -option java_outer_classname = "BKJournalProtos"; -option java_generate_equals_and_hash = true; -package hadoop.hdfs; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; -import "hdfs.proto"; -import "HdfsServer.proto"; +/** + * Aliyun OSS contract renaming tests. + */ +public class TestAliyunOSSContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); + } -message VersionProto { - required int32 layoutVersion = 1; - optional NamespaceInfoProto namespaceInfo = 2; } - -message EditLogLedgerProto { - required int32 dataLayoutVersion = 1; - required int64 ledgerId = 2; - required int64 firstTxId = 3; - optional int64 lastTxId = 4; -} - -message MaxTxIdProto { - required int64 txId = 1; -} - -message CurrentInprogressProto { - required string path = 1; - optional string hostname = 2; -} \ No newline at end of file diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java new file mode 100644 index 00000000000..9faae374521 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * Root dir operations against an Aliyun OSS bucket. + */ +public class TestAliyunOSSContractRootDir extends + AbstractContractRootDirectoryTest { + + private static final Logger LOG = + LoggerFactory.getLogger(TestAliyunOSSContractRootDir.class); + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); + } + + @Override + public void testListEmptyRootDirectory() throws IOException { + for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) { + try { + super.testListEmptyRootDirectory(); + break; + } catch (AssertionError | FileNotFoundException e) { + if (attempt < maxAttempts) { + LOG.info("Attempt {} of {} for empty root directory test failed. " + + "Attempting retry.", attempt, maxAttempts); + try { + Thread.sleep(1000); + } catch (InterruptedException e2) { + Thread.currentThread().interrupt(); + fail("Test interrupted."); + break; + } + } else { + LOG.error( + "Empty root directory test failed {} attempts. Failing test.", + maxAttempts); + throw e; + } + } + } + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java new file mode 100644 index 00000000000..b247ab10c7c --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Aliyun OSS contract seeking tests. + */ +public class TestAliyunOSSContractSeek extends AbstractContractSeekTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml new file mode 100644 index 00000000000..7bbbf4646b6 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml @@ -0,0 +1,115 @@ + + + + + + fs.contract.test.random-seek-count + 10 + + + + fs.contract.is-blobstore + true + + + + fs.contract.is-case-sensitive + true + + + + fs.contract.rename-returns-false-if-source-missing + false + + + + fs.contract.rename-remove-dest-if-empty-dir + false + + + + fs.contract.supports-append + false + + + + fs.contract.supports-atomic-directory-delete + false + + + + fs.contract.supports-atomic-rename + false + + + + fs.contract.supports-block-locality + false + + + + fs.contract.supports-concat + false + + + + fs.contract.supports-seek + true + + + + fs.contract.supports-seek-on-closed-file + true + + + + fs.contract.rejects-seek-past-eof + true + + + + fs.contract.supports-strict-exceptions + true + + + + fs.contract.supports-unix-permissions + false + + + + fs.contract.rename-overwrites-dest + true + + + + fs.contract.test.root-tests-enabled + true + + + + fs.contract.supports-getfilestatus + true + + + + fs.oss.multipart.download.size + 102400 + + diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml new file mode 100644 index 00000000000..fa4118c2162 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml @@ -0,0 +1,46 @@ + + + + + + + hadoop.tmp.dir + target/build/test + A base for other temporary directories. + true + + + + + hadoop.security.authentication + simple + + + + + + + + diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties new file mode 100644 index 00000000000..bb5cbe5ec32 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# log4j configuration used during build and unit tests + +log4j.rootLogger=INFO,stdout +log4j.threshold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 49b0379b96e..1c1bb023b46 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -285,6 +285,18 @@ aws-java-sdk-s3 compile + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-annotations + joda-time joda-time diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index e3b2c639076..85d1fc7e18c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -26,6 +26,7 @@ import java.io.InterruptedIOException; import java.net.URI; import java.util.ArrayList; import java.util.Date; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; @@ -59,16 +60,20 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.GlobalStorageStatistics; +import org.apache.hadoop.fs.InvalidRequestException; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.fs.permission.FsPermission; @@ -503,6 +508,31 @@ public class S3AFileSystem extends FileSystem { null); } + /** + * {@inheritDoc} + * @throws FileNotFoundException if the parent directory is not present -or + * is not a directory. + */ + @Override + public FSDataOutputStream createNonRecursive(Path path, + FsPermission permission, + EnumSet flags, + int bufferSize, + short replication, + long blockSize, + Progressable progress) throws IOException { + Path parent = path.getParent(); + if (parent != null) { + // expect this to raise an exception if there is no parent + if (!getFileStatus(parent).isDirectory()) { + throw new FileAlreadyExistsException("Not a directory: " + parent); + } + } + return create(path, permission, + flags.contains(CreateFlag.OVERWRITE), bufferSize, + replication, blockSize, progress); + } + /** * Append to an existing file (optional operation). * @param f the existing file to be appended. @@ -776,12 +806,26 @@ public class S3AFileSystem extends FileSystem { * operation statistics. * @param key key to blob to delete. */ - private void deleteObject(String key) { + private void deleteObject(String key) throws InvalidRequestException { + blockRootDelete(key); incrementWriteOperations(); incrementStatistic(OBJECT_DELETE_REQUESTS); s3.deleteObject(bucket, key); } + /** + * Reject any request to delete an object where the key is root. + * @param key key to validate + * @throws InvalidRequestException if the request was rejected due to + * a mistaken attempt to delete the root directory. + */ + private void blockRootDelete(String key) throws InvalidRequestException { + if (key.isEmpty() || "/".equals(key)) { + throw new InvalidRequestException("Bucket "+ bucket + +" cannot be deleted"); + } + } + /** * Perform a bulk object delete operation. * Increments the {@code OBJECT_DELETE_REQUESTS} and write @@ -921,17 +965,24 @@ public class S3AFileSystem extends FileSystem { /** * A helper method to delete a list of keys on a s3-backend. * - * @param keysToDelete collection of keys to delete on the s3-backend + * @param keysToDelete collection of keys to delete on the s3-backend. + * if empty, no request is made of the object store. * @param clearKeys clears the keysToDelete-list after processing the list * when set to true * @param deleteFakeDir indicates whether this is for deleting fake dirs + * @throws InvalidRequestException if the request was rejected due to + * a mistaken attempt to delete the root directory. */ private void removeKeys(List keysToDelete, - boolean clearKeys, boolean deleteFakeDir) throws AmazonClientException { + boolean clearKeys, boolean deleteFakeDir) + throws AmazonClientException, InvalidRequestException { if (keysToDelete.isEmpty()) { - // no keys + // exit fast if there are no keys to delete return; } + for (DeleteObjectsRequest.KeyVersion keyVersion : keysToDelete) { + blockRootDelete(keyVersion.getKey()); + } if (enableMultiObjectsDelete) { deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keysToDelete)); } else { @@ -993,18 +1044,16 @@ public class S3AFileSystem extends FileSystem { if (status.isDirectory()) { LOG.debug("delete: Path is a directory: {}", f); - if (!recursive && !status.isEmptyDirectory()) { - throw new IOException("Path is a folder: " + f + - " and it is not an empty directory"); - } - if (!key.endsWith("/")) { key = key + "/"; } if (key.equals("/")) { - LOG.info("s3a cannot delete the root directory"); - return false; + return rejectRootDirectoryDelete(status, recursive); + } + + if (!recursive && !status.isEmptyDirectory()) { + throw new PathIsNotEmptyDirectoryException(f.toString()); } if (status.isEmptyDirectory()) { @@ -1045,10 +1094,39 @@ public class S3AFileSystem extends FileSystem { deleteObject(key); } - createFakeDirectoryIfNecessary(f.getParent()); + Path parent = f.getParent(); + if (parent != null) { + createFakeDirectoryIfNecessary(parent); + } return true; } + /** + * Implements the specific logic to reject root directory deletion. + * The caller must return the result of this call, rather than + * attempt to continue with the delete operation: deleting root + * directories is never allowed. This method simply implements + * the policy of when to return an exit code versus raise an exception. + * @param status filesystem status + * @param recursive recursive flag from command + * @return a return code for the operation + * @throws PathIOException if the operation was explicitly rejected. + */ + private boolean rejectRootDirectoryDelete(S3AFileStatus status, + boolean recursive) throws IOException { + LOG.info("s3a delete the {} root directory of {}", bucket, recursive); + boolean emptyRoot = status.isEmptyDirectory(); + if (emptyRoot) { + return true; + } + if (recursive) { + return false; + } else { + // reject + throw new PathIOException(bucket, "Cannot delete root path"); + } + } + private void createFakeDirectoryIfNecessary(Path f) throws IOException, AmazonClientException { String key = pathToKey(f); @@ -1524,7 +1602,7 @@ public class S3AFileSystem extends FileSystem { } try { removeKeys(keysToRemove, false, true); - } catch(AmazonClientException e) { + } catch(AmazonClientException | InvalidRequestException e) { instrumentation.errorIgnored(); if (LOG.isDebugEnabled()) { StringBuilder sb = new StringBuilder(); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index a5e8e7a1b6a..93d819b72a9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -48,6 +48,7 @@ import java.util.concurrent.ExecutionException; import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY; import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER; +import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT; import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY; /** @@ -64,6 +65,7 @@ public final class S3AUtils { = "instantiation exception"; static final String NOT_AWS_PROVIDER = "does not implement AWSCredentialsProvider"; + static final String ENDPOINT_KEY = "Endpoint"; private S3AUtils() { } @@ -117,6 +119,21 @@ public final class S3AUtils { int status = ase.getStatusCode(); switch (status) { + case 301: + if (s3Exception != null) { + if (s3Exception.getAdditionalDetails() != null && + s3Exception.getAdditionalDetails().containsKey(ENDPOINT_KEY)) { + message = String.format("Received permanent redirect response to " + + "endpoint %s. This likely indicates that the S3 endpoint " + + "configured in %s does not match the AWS region containing " + + "the bucket.", + s3Exception.getAdditionalDetails().get(ENDPOINT_KEY), ENDPOINT); + } + ioe = new AWSS3IOException(message, s3Exception); + } else { + ioe = new AWSServiceIOException(message, ase); + } + break; // permissions case 401: case 403: diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 160aa46fcdc..cf785d5cf6a 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -1184,33 +1184,27 @@ As an example, the endpoint for S3 Frankfurt is `s3.eu-central-1.amazonaws.com`: ### Error message "The bucket you are attempting to access must be addressed using the specified endpoint" -This surfaces when `fs.s3a.endpoint` is configured to use S3 service endpoint +This surfaces when `fs.s3a.endpoint` is configured to use an S3 service endpoint which is neither the original AWS one, `s3.amazonaws.com` , nor the one where -the bucket is hosted. +the bucket is hosted. The error message contains the redirect target returned +by S3, which can be used to determine the correct value for `fs.s3a.endpoint`. ``` -org.apache.hadoop.fs.s3a.AWSS3IOException: purging multipart uploads on landsat-pds: - com.amazonaws.services.s3.model.AmazonS3Exception: - The bucket you are attempting to access must be addressed using the specified endpoint. - Please send all future requests to this endpoint. - (Service: Amazon S3; Status Code: 301; Error Code: PermanentRedirect; Request ID: 5B7A5D18BE596E4B), - S3 Extended Request ID: uE4pbbmpxi8Nh7rycS6GfIEi9UH/SWmJfGtM9IeKvRyBPZp/hN7DbPyz272eynz3PEMM2azlhjE=: - - at com.amazonaws.http.AmazonHttpClient.handleErrorResponse(AmazonHttpClient.java:1182) - at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:770) - at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:489) - at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:310) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3785) - at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3738) - at com.amazonaws.services.s3.AmazonS3Client.listMultipartUploads(AmazonS3Client.java:2796) - at com.amazonaws.services.s3.transfer.TransferManager.abortMultipartUploads(TransferManager.java:1217) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initMultipartUploads(S3AFileSystem.java:454) - at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:289) - at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2715) - at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:96) - at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2749) - at org.apache.hadoop.fs.FileSystem$Cache.getUnique(FileSystem.java:2737) - at org.apache.hadoop.fs.FileSystem.newInstance(FileSystem.java:430) +org.apache.hadoop.fs.s3a.AWSS3IOException: Received permanent redirect response + to bucket.s3-us-west-2.amazonaws.com. This likely indicates that the S3 + endpoint configured in fs.s3a.endpoint does not match the AWS region + containing the bucket.: The bucket you are attempting to access must be + addressed using the specified endpoint. Please send all future requests to + this endpoint. (Service: Amazon S3; Status Code: 301; + Error Code: PermanentRedirect; Request ID: 7D39EC1021C61B11) + at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:132) + at org.apache.hadoop.fs.s3a.S3AFileSystem.initMultipartUploads(S3AFileSystem.java:287) + at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:203) + at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2895) + at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:102) + at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2932) + at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2914) + at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:390) ``` 1. Use the [Specific endpoint of the bucket's S3 service](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java index d8e017e4a20..06864881e38 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java @@ -18,9 +18,6 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.model.AmazonS3Exception; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; @@ -34,7 +31,6 @@ import org.slf4j.LoggerFactory; import java.io.EOFException; import java.io.FileNotFoundException; -import java.nio.file.AccessDeniedException; import java.util.concurrent.Callable; import static org.apache.hadoop.fs.contract.ContractTestUtils.*; @@ -138,55 +134,4 @@ public class ITestS3AFailureHandling extends AbstractFSContractTestBase { assertEquals("Expected EOF from "+ operation + "; got char " + (char) readResult, -1, readResult); } - - @Test - public void test404isNotFound() throws Throwable { - verifyTranslated(FileNotFoundException.class, createS3Exception(404)); - } - - protected Exception verifyTranslated(Class clazz, - AmazonClientException exception) throws Exception { - return verifyExceptionClass(clazz, - translateException("test", "/", exception)); - } - - @Test - public void test401isNotPermittedFound() throws Throwable { - verifyTranslated(AccessDeniedException.class, - createS3Exception(401)); - } - - protected AmazonS3Exception createS3Exception(int code) { - AmazonS3Exception source = new AmazonS3Exception(""); - source.setStatusCode(code); - return source; - } - - @Test - public void testGenericS3Exception() throws Throwable { - // S3 exception of no known type - AWSS3IOException ex = (AWSS3IOException)verifyTranslated( - AWSS3IOException.class, - createS3Exception(451)); - assertEquals(451, ex.getStatusCode()); - } - - @Test - public void testGenericServiceS3Exception() throws Throwable { - // service exception of no known type - AmazonServiceException ase = new AmazonServiceException("unwind"); - ase.setStatusCode(500); - AWSServiceIOException ex = (AWSServiceIOException)verifyTranslated( - AWSServiceIOException.class, - ase); - assertEquals(500, ex.getStatusCode()); - } - - @Test - public void testGenericClientException() throws Throwable { - // Generic Amazon exception - verifyTranslated(AWSClientIOException.class, - new AmazonClientException("")); - } - } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java new file mode 100644 index 00000000000..59fcb05729c --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * Tests of the S3A FileSystem which don't have a specific home and can share + * a filesystem instance with others.. + */ +public class ITestS3AMiscOperations extends AbstractS3ATestBase { + + @Test + public void testCreateNonRecursiveSuccess() throws IOException { + Path shouldWork = path("nonrecursivenode"); + try(FSDataOutputStream out = createNonRecursive(shouldWork)) { + out.write(0); + out.close(); + } + assertIsFile(shouldWork); + } + + @Test(expected = FileNotFoundException.class) + public void testCreateNonRecursiveNoParent() throws IOException { + createNonRecursive(path("/recursive/node")); + } + + @Test(expected = FileAlreadyExistsException.class) + public void testCreateNonRecursiveParentIsFile() throws IOException { + Path parent = path("/file.txt"); + ContractTestUtils.touch(getFileSystem(), parent); + createNonRecursive(new Path(parent, "fail")); + } + + private FSDataOutputStream createNonRecursive(Path path) throws IOException { + return getFileSystem().createNonRecursive(path, false, 4096, + (short) 3, (short) 4096, + null); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java new file mode 100644 index 00000000000..a7dafa06c18 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import static org.apache.hadoop.fs.s3a.Constants.*; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; +import static org.apache.hadoop.fs.s3a.S3AUtils.*; +import static org.junit.Assert.*; + +import java.io.EOFException; +import java.io.FileNotFoundException; +import java.nio.file.AccessDeniedException; +import java.util.Collections; +import java.util.Map; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.model.AmazonS3Exception; + +import org.junit.Test; + +/** + * Unit test suite covering translation of AWS SDK exceptions to S3A exceptions. + */ +public class TestS3AExceptionTranslation { + + @Test + public void test301ContainsEndpoint() throws Exception { + AmazonS3Exception s3Exception = createS3Exception("wrong endpoint", 301, + Collections.singletonMap(S3AUtils.ENDPOINT_KEY, + "bucket.s3-us-west-2.amazonaws.com")); + AWSS3IOException ex = (AWSS3IOException)verifyTranslated( + AWSS3IOException.class, s3Exception); + assertEquals(301, ex.getStatusCode()); + assertNotNull(ex.getMessage()); + assertTrue(ex.getMessage().contains("bucket.s3-us-west-2.amazonaws.com")); + assertTrue(ex.getMessage().contains(ENDPOINT)); + } + + @Test + public void test401isNotPermittedFound() throws Exception { + verifyTranslated(AccessDeniedException.class, + createS3Exception(401)); + } + + @Test + public void test403isNotPermittedFound() throws Exception { + verifyTranslated(AccessDeniedException.class, + createS3Exception(403)); + } + + @Test + public void test404isNotFound() throws Exception { + verifyTranslated(FileNotFoundException.class, createS3Exception(404)); + } + + @Test + public void test410isNotFound() throws Exception { + verifyTranslated(FileNotFoundException.class, createS3Exception(410)); + } + + @Test + public void test416isEOF() throws Exception { + verifyTranslated(EOFException.class, createS3Exception(416)); + } + + @Test + public void testGenericS3Exception() throws Exception { + // S3 exception of no known type + AWSS3IOException ex = (AWSS3IOException)verifyTranslated( + AWSS3IOException.class, + createS3Exception(451)); + assertEquals(451, ex.getStatusCode()); + } + + @Test + public void testGenericServiceS3Exception() throws Exception { + // service exception of no known type + AmazonServiceException ase = new AmazonServiceException("unwind"); + ase.setStatusCode(500); + AWSServiceIOException ex = (AWSServiceIOException)verifyTranslated( + AWSServiceIOException.class, + ase); + assertEquals(500, ex.getStatusCode()); + } + + @Test + public void testGenericClientException() throws Exception { + // Generic Amazon exception + verifyTranslated(AWSClientIOException.class, + new AmazonClientException("")); + } + + private static AmazonS3Exception createS3Exception(int code) { + return createS3Exception("", code, null); + } + + private static AmazonS3Exception createS3Exception(String message, int code, + Map additionalDetails) { + AmazonS3Exception source = new AmazonS3Exception(message); + source.setStatusCode(code); + source.setAdditionalDetails(additionalDetails); + return source; + } + + private static Exception verifyTranslated(Class clazz, + AmazonClientException exception) throws Exception { + return verifyExceptionClass(clazz, + translateException("test", "/", exception)); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml index c07a1d7adbc..e1a0bfeaccd 100644 --- a/hadoop-tools/hadoop-azure-datalake/pom.xml +++ b/hadoop-tools/hadoop-azure-datalake/pom.xml @@ -181,5 +181,9 @@ 2.4.0 test + + com.fasterxml.jackson.core + jackson-databind + diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/oauth2/AzureADClientCredentialBasedAccesTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/oauth2/AzureADClientCredentialBasedAccesTokenProvider.java index 6dfc593feaa..11d07e70e96 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/oauth2/AzureADClientCredentialBasedAccesTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/oauth2/AzureADClientCredentialBasedAccesTokenProvider.java @@ -18,6 +18,9 @@ */ package org.apache.hadoop.hdfs.web.oauth2; +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.fasterxml.jackson.databind.ObjectReader; import com.squareup.okhttp.OkHttpClient; import com.squareup.okhttp.Request; import com.squareup.okhttp.RequestBody; @@ -29,8 +32,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.web.URLConnectionFactory; import org.apache.hadoop.util.Timer; import org.apache.http.HttpStatus; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.ObjectReader; import java.io.IOException; import java.util.Map; diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 48f9043ae16..d8121e26959 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -217,6 +217,10 @@ mockito-all test - + + com.fasterxml.jackson.core + jackson-databind + + diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index fb0d31f330b..54eb90f45d0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -40,6 +40,10 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -63,14 +67,10 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Time; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.JsonParseException; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.map.JsonMappingException; -import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.microsoft.azure.storage.StorageException; @@ -193,8 +193,8 @@ public class NativeAzureFileSystem extends FileSystem { if (oldFolderName == null || newFolderName == null) { this.committed = false; } else { - this.srcKey = oldFolderName.getTextValue(); - this.dstKey = newFolderName.getTextValue(); + this.srcKey = oldFolderName.textValue(); + this.dstKey = newFolderName.textValue(); if (this.srcKey == null || this.dstKey == null) { this.committed = false; } else { @@ -203,7 +203,7 @@ public class NativeAzureFileSystem extends FileSystem { this.committed = false; } else { for (int i = 0; i < fileList.size(); i++) { - fileStrList.add(fileList.get(i).getTextValue()); + fileStrList.add(fileList.get(i).textValue()); } } } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java index 96f364c4b40..6171aa9b991 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java @@ -18,6 +18,8 @@ package org.apache.hadoop.tools; * limitations under the License. */ +import org.apache.hadoop.fs.Path; + /** * Utility class to hold commonly used constants. */ @@ -124,10 +126,18 @@ public class DistCpConstants { public static final int MIN_RECORDS_PER_CHUNK_DEFAULT = 5; public static final int SPLIT_RATIO_DEFAULT = 2; + /** + * Constants for NONE file deletion + */ + public static final String NONE_PATH_NAME = "/NONE"; + public static final Path NONE_PATH = new Path(NONE_PATH_NAME); + public static final Path RAW_NONE_PATH = new Path( + DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME + NONE_PATH_NAME); + /** * Value of reserved raw HDFS directory when copying raw.* xattrs. */ - static final String HDFS_RESERVED_RAW_DIRECTORY_NAME = "/.reserved/raw"; + public static final String HDFS_RESERVED_RAW_DIRECTORY_NAME = "/.reserved/raw"; static final String HDFS_DISTCP_DIFF_DIRECTORY_NAME = ".distcp.diff.tmp"; } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java index 6d2fef5f907..dd653b297df 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java @@ -238,7 +238,10 @@ public class CopyCommitter extends FileOutputCommitter { List targets = new ArrayList(1); Path targetFinalPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH)); targets.add(targetFinalPath); - DistCpOptions options = new DistCpOptions(targets, new Path("/NONE")); + Path resultNonePath = Path.getPathWithoutSchemeAndAuthority(targetFinalPath) + .toString().startsWith(DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME) + ? DistCpConstants.RAW_NONE_PATH : DistCpConstants.NONE_PATH; + DistCpOptions options = new DistCpOptions(targets, resultNonePath); // // Set up options to be the same from the CopyListing.buildListing's perspective, // so to collect similar listings as when doing the copy diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java index 5aef51a830e..8adc2cfb867 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java @@ -82,14 +82,7 @@ public class TestDistCpWithRawXAttrs { final String relDst = "/./.reserved/../.reserved/raw/../raw/dest/../dest"; doTestPreserveRawXAttrs(relSrc, relDst, "-px", true, true, DistCpConstants.SUCCESS); - doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-px", - false, true, DistCpConstants.SUCCESS); - doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-px", - false, true, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-px", - false, true, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-px", - true, true, DistCpConstants.SUCCESS); + doTestStandardPreserveRawXAttrs("-px", true); final Path savedWd = fs.getWorkingDirectory(); try { fs.setWorkingDirectory(new Path("/.reserved/raw")); @@ -103,27 +96,18 @@ public class TestDistCpWithRawXAttrs { /* Test that XAttrs are not preserved and raw.* are when appropriate. */ @Test public void testPreserveRawXAttrs2() throws Exception { - doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-p", - false, false, DistCpConstants.SUCCESS); - doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-p", - false, false, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-p", - false, false, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-p", - true, false, DistCpConstants.SUCCESS); + doTestStandardPreserveRawXAttrs("-p", false); } /* Test that XAttrs are not preserved and raw.* are when appropriate. */ @Test public void testPreserveRawXAttrs3() throws Exception { - doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, null, - false, false, DistCpConstants.SUCCESS); - doTestPreserveRawXAttrs(rootedSrcName, rawDestName, null, - false, false, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rootedDestName, null, - false, false, DistCpConstants.INVALID_ARGUMENT); - doTestPreserveRawXAttrs(rawSrcName, rawDestName, null, - true, false, DistCpConstants.SUCCESS); + doTestStandardPreserveRawXAttrs(null, false); + } + + @Test + public void testPreserveRawXAttrs4() throws Exception { + doTestStandardPreserveRawXAttrs("-update -delete", false); } private static Path[] pathnames = { new Path("dir1"), @@ -145,6 +129,19 @@ public class TestDistCpWithRawXAttrs { } } + private void doTestStandardPreserveRawXAttrs(String options, + boolean expectUser) + throws Exception { + doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, options, + false, expectUser, DistCpConstants.SUCCESS); + doTestPreserveRawXAttrs(rootedSrcName, rawDestName, options, + false, expectUser, DistCpConstants.INVALID_ARGUMENT); + doTestPreserveRawXAttrs(rawSrcName, rootedDestName, options, + false, expectUser, DistCpConstants.INVALID_ARGUMENT); + doTestPreserveRawXAttrs(rawSrcName, rawDestName, options, + true, expectUser, DistCpConstants.SUCCESS); + } + private void doTestPreserveRawXAttrs(String src, String dest, String preserveOpts, boolean expectRaw, boolean expectUser, int expectedExitCode) throws Exception { diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java index 27216381d1b..624f7d5a0ef 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java @@ -18,20 +18,19 @@ package org.apache.hadoop.tools.util; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.tools.DistCp; +import org.apache.hadoop.util.ToolRunner; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -import org.apache.hadoop.tools.DistCp; -import org.apache.hadoop.util.ToolRunner; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * Utility class for DistCpTests @@ -79,10 +78,19 @@ public class DistCpTestUtils { public static void assertRunDistCp(int exitCode, String src, String dst, String options, Configuration conf) throws Exception { + assertRunDistCp(exitCode, src, dst, + options == null ? new String[0] : options.trim().split(" "), conf); + } + + private static void assertRunDistCp(int exitCode, String src, String dst, + String[] options, Configuration conf) + throws Exception { DistCp distCp = new DistCp(conf, null); - String[] optsArr = options == null ? - new String[] { src, dst } : - new String[] { options, src, dst }; + String[] optsArr = new String[options.length + 2]; + System.arraycopy(options, 0, optsArr, 0, options.length); + optsArr[optsArr.length - 2] = src; + optsArr[optsArr.length - 1] = dst; + assertEquals(exitCode, ToolRunner.run(conf, distCp, optsArr)); } diff --git a/hadoop-tools/hadoop-kafka/pom.xml b/hadoop-tools/hadoop-kafka/pom.xml index ea77979e8b0..8d64889163b 100644 --- a/hadoop-tools/hadoop-kafka/pom.xml +++ b/hadoop-tools/hadoop-kafka/pom.xml @@ -125,7 +125,7 @@ org.apache.kafka - kafka_2.10 + kafka-clients ${kafka.version} diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml index b036e8455a0..f9a4df0019e 100644 --- a/hadoop-tools/hadoop-openstack/pom.xml +++ b/hadoop-tools/hadoop-openstack/pom.xml @@ -123,16 +123,6 @@ compile - - org.codehaus.jackson - jackson-mapper-asl - compile - - - org.codehaus.jackson - jackson-core-asl - compile - commons-httpclient commons-httpclient @@ -150,5 +140,13 @@ junit provided + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-databind + diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyAuthenticationRequest.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyAuthenticationRequest.java index f5f9a8cfc3f..e25d17d2fb8 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyAuthenticationRequest.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyAuthenticationRequest.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.swift.auth; -import org.codehaus.jackson.annotate.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty; /** * Class that represents authentication request to Openstack Keystone. diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/AccessToken.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/AccessToken.java index a01e8553008..b38d4660e5a 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/AccessToken.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/AccessToken.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.swift.auth.entities; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; /** * Access token representation of Openstack Keystone authentication. diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Catalog.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Catalog.java index 838d87f9dd7..76e161b0642 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Catalog.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Catalog.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.swift.auth.entities; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.List; diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Endpoint.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Endpoint.java index f9de89518f8..b1cbf2acc7b 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Endpoint.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Endpoint.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.swift.auth.entities; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.net.URI; diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Tenant.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Tenant.java index da94c402e21..405d2c85368 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Tenant.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Tenant.java @@ -18,7 +18,7 @@ package org.apache.hadoop.fs.swift.auth.entities; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; /** * Tenant is abstraction in Openstack which describes all account diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/User.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/User.java index 1a6954a5cb4..da3bac20f2b 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/User.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/User.java @@ -18,8 +18,8 @@ package org.apache.hadoop.fs.swift.auth.entities; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.hadoop.fs.swift.auth.Roles; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; import java.util.List; diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java index cc3e3d2f72a..71d8d824ddd 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.fs.swift.snative; +import com.fasterxml.jackson.databind.type.CollectionType; + import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.logging.Log; @@ -36,7 +38,6 @@ import org.apache.hadoop.fs.swift.util.DurationStats; import org.apache.hadoop.fs.swift.util.JSONUtil; import org.apache.hadoop.fs.swift.util.SwiftObjectPath; import org.apache.hadoop.fs.swift.util.SwiftUtils; -import org.codehaus.jackson.map.type.CollectionType; import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java index b17cb6590c1..fee7e7f5697 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java @@ -18,12 +18,12 @@ package org.apache.hadoop.fs.swift.util; +import com.fasterxml.jackson.core.JsonGenerationException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.CollectionType; import org.apache.hadoop.fs.swift.exceptions.SwiftJsonMarshallingException; -import org.codehaus.jackson.JsonGenerationException; -import org.codehaus.jackson.map.JsonMappingException; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.type.CollectionType; -import org.codehaus.jackson.type.TypeReference; import java.io.IOException; import java.io.StringWriter; @@ -54,9 +54,7 @@ public class JSONUtil { try { jsonMapper.writeValue(json, object); return json.toString(); - } catch (JsonGenerationException e) { - throw new SwiftJsonMarshallingException(e.toString(), e); - } catch (JsonMappingException e) { + } catch (JsonGenerationException | JsonMappingException e) { throw new SwiftJsonMarshallingException(e.toString(), e); } } @@ -96,9 +94,7 @@ public class JSONUtil { throws IOException { try { return (T)jsonMapper.readValue(value, typeReference); - } catch (JsonGenerationException e) { - throw new SwiftJsonMarshallingException("Error generating response", e); - } catch (JsonMappingException e) { + } catch (JsonGenerationException | JsonMappingException e) { throw new SwiftJsonMarshallingException("Error generating response", e); } } @@ -115,11 +111,7 @@ public class JSONUtil { throws IOException { try { return (T)jsonMapper.readValue(value, collectionType); - } catch (JsonGenerationException e) { - throw new SwiftJsonMarshallingException(e.toString() - + " source: " + value, - e); - } catch (JsonMappingException e) { + } catch (JsonGenerationException | JsonMappingException e) { throw new SwiftJsonMarshallingException(e.toString() + " source: " + value, e); diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml index f5acf4e2a5c..9e1b1f34ae2 100644 --- a/hadoop-tools/hadoop-rumen/pom.xml +++ b/hadoop-tools/hadoop-rumen/pom.xml @@ -94,6 +94,15 @@ test-jar test + + com.fasterxml.jackson.core + jackson-databind + test + + + com.fasterxml.jackson.core + jackson-databind + diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Anonymizer.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Anonymizer.java index e1a2be7ce88..3c85a93ddbf 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Anonymizer.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Anonymizer.java @@ -21,6 +21,12 @@ package org.apache.hadoop.tools.rumen; import java.io.IOException; import java.io.OutputStream; +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.Version; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; @@ -36,13 +42,6 @@ import org.apache.hadoop.tools.rumen.datatypes.*; import org.apache.hadoop.tools.rumen.serializers.*; import org.apache.hadoop.tools.rumen.state.*; -import org.codehaus.jackson.JsonEncoding; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.Version; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.module.SimpleModule; - public class Anonymizer extends Configured implements Tool { private boolean anonymizeTrace = false; private Path inputTracePath = null; @@ -88,8 +87,8 @@ public class Anonymizer extends Configured implements Tool { outMapper = new ObjectMapper(); // define a module - SimpleModule module = new SimpleModule("Anonymization Serializer", - new Version(0, 1, 1, "FINAL")); + SimpleModule module = new SimpleModule( + "Anonymization Serializer", new Version(0, 1, 1, "FINAL", "", "")); // add various serializers to the module // use the default (as-is) serializer for default data types module.addSerializer(DataType.class, new DefaultRumenSerializer()); @@ -106,7 +105,7 @@ public class Anonymizer extends Configured implements Tool { // register the module with the object-mapper outMapper.registerModule(module); - outFactory = outMapper.getJsonFactory(); + outFactory = outMapper.getFactory(); } // anonymize the job trace file @@ -191,8 +190,8 @@ public class Anonymizer extends Configured implements Tool { output = outFS.create(path); } - JsonGenerator outGen = outFactory.createJsonGenerator(output, - JsonEncoding.UTF8); + JsonGenerator outGen = + outFactory.createGenerator(output, JsonEncoding.UTF8); outGen.useDefaultPrettyPrinter(); return outGen; diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java index c53a7c2ddd0..eceb98d2cc2 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -57,8 +58,6 @@ import org.apache.hadoop.io.compress.CompressionCodecFactory; import org.apache.hadoop.io.compress.CodecPool; import org.apache.hadoop.io.compress.Decompressor; -import org.codehaus.jackson.JsonProcessingException; - /** * This is the main class for rumen log mining functionality. * diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JsonObjectMapperParser.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JsonObjectMapperParser.java index cbd36793b23..f95878dde95 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JsonObjectMapperParser.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JsonObjectMapperParser.java @@ -18,15 +18,14 @@ package org.apache.hadoop.tools.rumen; import java.io.Closeable; -import java.io.EOFException; import java.io.IOException; import java.io.InputStream; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; /** * A simple wrapper for parsing JSON-encoded data using ObjectMapper. @@ -50,11 +49,9 @@ class JsonObjectMapperParser implements Closeable { public JsonObjectMapperParser(Path path, Class clazz, Configuration conf) throws IOException { mapper = new ObjectMapper(); - mapper.configure( - DeserializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true); this.clazz = clazz; InputStream input = new PossiblyDecompressedInputStream(path, conf); - jsonParser = mapper.getJsonFactory().createJsonParser(input); + jsonParser = mapper.getFactory().createParser(input); } /** @@ -66,10 +63,8 @@ class JsonObjectMapperParser implements Closeable { public JsonObjectMapperParser(InputStream input, Class clazz) throws IOException { mapper = new ObjectMapper(); - mapper.configure( - DeserializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true); this.clazz = clazz; - jsonParser = mapper.getJsonFactory().createJsonParser(input); + jsonParser = mapper.getFactory().createParser(input); } /** @@ -82,7 +77,7 @@ class JsonObjectMapperParser implements Closeable { public T getNext() throws IOException { try { return mapper.readValue(jsonParser, clazz); - } catch (EOFException e) { + } catch (JsonMappingException e) { return null; } } diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JsonObjectMapperWriter.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JsonObjectMapperWriter.java index 47bfee0e7ff..747b141fd98 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JsonObjectMapperWriter.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JsonObjectMapperWriter.java @@ -21,16 +21,15 @@ import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.Version; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; import org.apache.hadoop.mapreduce.ID; import org.apache.hadoop.tools.rumen.datatypes.DataType; import org.apache.hadoop.tools.rumen.serializers.DefaultRumenSerializer; import org.apache.hadoop.tools.rumen.serializers.ObjectStringSerializer; -import org.codehaus.jackson.JsonEncoding; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.Version; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.SerializationConfig; -import org.codehaus.jackson.map.module.SimpleModule; /** * Simple wrapper around {@link JsonGenerator} to write objects in JSON format. @@ -41,12 +40,10 @@ public class JsonObjectMapperWriter implements Closeable { public JsonObjectMapperWriter(OutputStream output, boolean prettyPrint) throws IOException { ObjectMapper mapper = new ObjectMapper(); - mapper.configure( - SerializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true); // define a module - SimpleModule module = new SimpleModule("Default Serializer", - new Version(0, 1, 1, "FINAL")); + SimpleModule module = new SimpleModule( + "Default Serializer", new Version(0, 1, 1, "FINAL", "", "")); // add various serializers to the module // add default (all-pass) serializer for all rumen specific data types module.addSerializer(DataType.class, new DefaultRumenSerializer()); @@ -56,9 +53,7 @@ public class JsonObjectMapperWriter implements Closeable { // register the module with the object-mapper mapper.registerModule(module); - mapper.getJsonFactory(); - writer = mapper.getJsonFactory().createJsonGenerator( - output, JsonEncoding.UTF8); + writer = mapper.getFactory().createGenerator(output, JsonEncoding.UTF8); if (prettyPrint) { writer.useDefaultPrettyPrinter(); } diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedJob.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedJob.java index 785feb31325..597aab88deb 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedJob.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedJob.java @@ -27,9 +27,9 @@ import java.util.Properties; import java.util.Set; import java.util.TreeSet; +import com.fasterxml.jackson.annotation.JsonAnySetter; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.tools.rumen.datatypes.*; -import org.codehaus.jackson.annotate.JsonAnySetter; /** * A {@link LoggedDiscreteCDF} is a representation of an hadoop job, with the diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedLocation.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedLocation.java index 047cd63a907..6d51e4a8651 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedLocation.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedLocation.java @@ -25,8 +25,8 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; +import com.fasterxml.jackson.annotation.JsonAnySetter; import org.apache.hadoop.tools.rumen.datatypes.NodeName; -import org.codehaus.jackson.annotate.JsonAnySetter; /** * A {@link LoggedLocation} is a representation of a point in an hierarchical diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java index 23bbb98bb00..5d79a83c343 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java @@ -29,8 +29,8 @@ import java.util.TreeSet; import java.util.ArrayList; import java.util.Comparator; +import com.fasterxml.jackson.annotation.JsonAnySetter; import org.apache.hadoop.tools.rumen.datatypes.NodeName; -import org.codehaus.jackson.annotate.JsonAnySetter; /** * A {@link LoggedNetworkTopology} represents a tree that in turn represents a diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java index e507116e457..d9be325b2f4 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java @@ -17,11 +17,11 @@ */ package org.apache.hadoop.tools.rumen; +import com.fasterxml.jackson.annotation.JsonAnySetter; + import java.util.Set; import java.util.TreeSet; -import org.codehaus.jackson.annotate.JsonAnySetter; - /** * A {@link LoggedSingleRelativeRanking} represents an X-Y coordinate of a * single point in a discrete CDF. diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java index 4a23fa6fcc1..4ae33a76617 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java @@ -23,13 +23,13 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; +import com.fasterxml.jackson.annotation.JsonAnySetter; import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.jobhistory.JhCounter; import org.apache.hadoop.mapreduce.jobhistory.JhCounterGroup; import org.apache.hadoop.mapreduce.jobhistory.JhCounters; import org.apache.hadoop.util.StringUtils; -import org.codehaus.jackson.annotate.JsonAnySetter; /** * A {@link LoggedTask} represents a [hadoop] task that is part of a hadoop job. diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java index c21eb396635..5c6abd372c0 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java @@ -23,8 +23,8 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; +import com.fasterxml.jackson.annotation.JsonAnySetter; import org.apache.hadoop.util.StringUtils; -import org.codehaus.jackson.annotate.JsonAnySetter; // HACK ALERT!!! This "should" have have two subclasses, which might be called // LoggedMapTaskAttempt and LoggedReduceTaskAttempt, but diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/NodeName.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/NodeName.java index c0b8d45cf75..20eb535d0cb 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/NodeName.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/NodeName.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.tools.rumen.datatypes; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.tools.rumen.ParsedHost; import org.apache.hadoop.tools.rumen.anonymization.WordList; import org.apache.hadoop.tools.rumen.state.State; import org.apache.hadoop.tools.rumen.state.StatePool; -import org.codehaus.jackson.annotate.JsonIgnore; /** * Represents the cluster host. diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/BlockingSerializer.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/BlockingSerializer.java index 4338602f9c5..a720214ec82 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/BlockingSerializer.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/BlockingSerializer.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.tools.rumen.serializers; -import java.io.IOException; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.JsonProcessingException; -import org.codehaus.jackson.map.JsonSerializer; -import org.codehaus.jackson.map.SerializerProvider; +import java.io.IOException; /** * A JSON serializer for Strings. diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/DefaultAnonymizingRumenSerializer.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/DefaultAnonymizingRumenSerializer.java index d4e6fd55338..944d1932ba4 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/DefaultAnonymizingRumenSerializer.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/DefaultAnonymizingRumenSerializer.java @@ -19,13 +19,13 @@ package org.apache.hadoop.tools.rumen.serializers; import java.io.IOException; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.tools.rumen.datatypes.AnonymizableDataType; import org.apache.hadoop.tools.rumen.state.StatePool; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.JsonProcessingException; -import org.codehaus.jackson.map.JsonSerializer; -import org.codehaus.jackson.map.SerializerProvider; /** * Default Rumen JSON serializer. diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/DefaultRumenSerializer.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/DefaultRumenSerializer.java index 1b433d88463..766a75057be 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/DefaultRumenSerializer.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/DefaultRumenSerializer.java @@ -19,11 +19,12 @@ package org.apache.hadoop.tools.rumen.serializers; import java.io.IOException; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + import org.apache.hadoop.tools.rumen.datatypes.DataType; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.JsonProcessingException; -import org.codehaus.jackson.map.JsonSerializer; -import org.codehaus.jackson.map.SerializerProvider; /** * Default Rumen JSON serializer. diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/ObjectStringSerializer.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/ObjectStringSerializer.java index 69e8950d830..a576871a12f 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/ObjectStringSerializer.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/serializers/ObjectStringSerializer.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.tools.rumen.serializers; -import java.io.IOException; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.JsonProcessingException; -import org.codehaus.jackson.map.JsonSerializer; -import org.codehaus.jackson.map.SerializerProvider; +import java.io.IOException; /** * Rumen JSON serializer for serializing object using toSring() API. diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/State.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/State.java index 94a78c29da8..6ff43d5d26a 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/State.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/State.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.tools.rumen.state; -import org.codehaus.jackson.annotate.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnore; /** * Represents a state. This state is managed by {@link StatePool}. diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java index 47ceb8e8a70..4fd2f12f65c 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java @@ -19,13 +19,13 @@ package org.apache.hadoop.tools.rumen.state; import java.io.IOException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.hadoop.tools.rumen.state.StatePool.StatePair; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonProcessingException; -import org.codehaus.jackson.map.DeserializationContext; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.deser.std.StdDeserializer; -import org.codehaus.jackson.node.ObjectNode; /** * Rumen JSON deserializer for deserializing the {@link State} object. @@ -46,7 +46,7 @@ public class StateDeserializer extends StdDeserializer { try { stateClass = - Class.forName(statePairObject.get("className").getTextValue().trim()); + Class.forName(statePairObject.get("className").textValue().trim()); } catch (ClassNotFoundException cnfe) { throw new RuntimeException("Invalid classname!", cnfe); } diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java index 576a3c0edcf..6e6c8591e6c 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java @@ -27,6 +27,14 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.HashMap; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.Version; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -35,16 +43,6 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.tools.rumen.Anonymizer; import org.apache.hadoop.tools.rumen.datatypes.DataType; -import org.codehaus.jackson.JsonEncoding; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.Version; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.SerializationConfig; -import org.codehaus.jackson.map.module.SimpleModule; /** * A pool of states. States used by {@link DataType}'s can be managed the @@ -212,20 +210,16 @@ public class StatePool { private void read(DataInput in) throws IOException { ObjectMapper mapper = new ObjectMapper(); - mapper.configure( - DeserializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true); - // define a module SimpleModule module = new SimpleModule("State Serializer", - new Version(0, 1, 1, "FINAL")); + new Version(0, 1, 1, "FINAL", "", "")); // add the state deserializer module.addDeserializer(StatePair.class, new StateDeserializer()); // register the module with the object-mapper mapper.registerModule(module); - JsonParser parser = - mapper.getJsonFactory().createJsonParser((DataInputStream)in); + JsonParser parser = mapper.getFactory().createParser((DataInputStream)in); StatePool statePool = mapper.readValue(parser, StatePool.class); this.setStates(statePool.getStates()); parser.close(); @@ -283,20 +277,18 @@ public class StatePool { // This is just a JSON experiment System.out.println("Dumping the StatePool's in JSON format."); ObjectMapper outMapper = new ObjectMapper(); - outMapper.configure( - SerializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true); // define a module SimpleModule module = new SimpleModule("State Serializer", - new Version(0, 1, 1, "FINAL")); + new Version(0, 1, 1, "FINAL", "", "")); // add the state serializer //module.addSerializer(State.class, new StateSerializer()); // register the module with the object-mapper outMapper.registerModule(module); - JsonFactory outFactory = outMapper.getJsonFactory(); - JsonGenerator jGen = - outFactory.createJsonGenerator((DataOutputStream)out, JsonEncoding.UTF8); + JsonFactory outFactory = outMapper.getFactory(); + JsonGenerator jGen = + outFactory.createGenerator((DataOutputStream)out, JsonEncoding.UTF8); jGen.useDefaultPrettyPrinter(); jGen.writeObject(this); diff --git a/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java b/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java index 372d93e1062..206095a2576 100644 --- a/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java +++ b/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java @@ -21,16 +21,17 @@ import java.io.IOException; import java.util.List; +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.codehaus.jackson.JsonEncoding; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.map.ObjectMapper; import org.junit.Ignore; import org.junit.Test; @@ -139,9 +140,9 @@ public class TestHistograms { Path goldFilePath = new Path(filePath.getParent(), "gold"+testName); ObjectMapper mapper = new ObjectMapper(); - JsonFactory factory = mapper.getJsonFactory(); + JsonFactory factory = mapper.getFactory(); FSDataOutputStream ostream = lfs.create(goldFilePath, true); - JsonGenerator gen = factory.createJsonGenerator(ostream, + JsonGenerator gen = factory.createGenerator(ostream, JsonEncoding.UTF8); gen.useDefaultPrettyPrinter(); diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml index e361ba38ca9..da70b245936 100644 --- a/hadoop-tools/hadoop-sls/pom.xml +++ b/hadoop-tools/hadoop-sls/pom.xml @@ -70,6 +70,10 @@ jetty-util provided + + com.fasterxml.jackson.core + jackson-databind + diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java index 0d0745c4137..76bcb157188 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java @@ -34,6 +34,9 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; @@ -41,9 +44,6 @@ import org.apache.commons.cli.Options; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.sls.utils.SLSUtils; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.ObjectWriter; @Private @Unstable @@ -127,7 +127,7 @@ public class RumenToSLSConverter { ObjectMapper mapper = new ObjectMapper(); ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter(); Iterator i = mapper.readValues( - new JsonFactory().createJsonParser(input), Map.class); + new JsonFactory().createParser(input), Map.class); while (i.hasNext()) { Map m = i.next(); output.write(writer.writeValueAsString(createSLSJob(m)) + EOL); diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index c9c5c3847db..61738fb5340 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -33,6 +33,9 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; + import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; @@ -66,8 +69,6 @@ import org.apache.hadoop.yarn.sls.scheduler.TaskRunner; import org.apache.hadoop.yarn.sls.utils.SLSUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.log4j.Logger; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.map.ObjectMapper; @Private @Unstable @@ -281,7 +282,7 @@ public class SLSRunner { Reader input = new InputStreamReader(new FileInputStream(inputTrace), "UTF-8"); try { - Iterator i = mapper.readValues(jsonF.createJsonParser(input), + Iterator i = mapper.readValues(jsonF.createParser(input), Map.class); while (i.hasNext()) { Map jsonJob = i.next(); diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java index f1b4f078028..e5f7cd067b5 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java @@ -28,6 +28,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -37,8 +39,6 @@ import org.apache.hadoop.tools.rumen.JobTraceReader; import org.apache.hadoop.tools.rumen.LoggedJob; import org.apache.hadoop.tools.rumen.LoggedTask; import org.apache.hadoop.tools.rumen.LoggedTaskAttempt; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.map.ObjectMapper; @Private @Unstable @@ -106,8 +106,7 @@ public class SLSUtils { Reader input = new InputStreamReader(new FileInputStream(jobTrace), "UTF-8"); try { - Iterator i = mapper.readValues( - jsonF.createJsonParser(input), Map.class); + Iterator i = mapper.readValues(jsonF.createParser(input), Map.class); while (i.hasNext()) { Map jsonE = i.next(); List tasks = (List) jsonE.get("job.tasks"); @@ -134,8 +133,7 @@ public class SLSUtils { Reader input = new InputStreamReader(new FileInputStream(nodeFile), "UTF-8"); try { - Iterator i = mapper.readValues( - jsonF.createJsonParser(input), Map.class); + Iterator i = mapper.readValues(jsonF.createParser(input), Map.class); while (i.hasNext()) { Map jsonE = i.next(); String rack = "/" + jsonE.get("rack"); diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index 899a9455d29..14fa9f05f78 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -100,6 +100,12 @@ compile ${project.version} + + org.apache.hadoop + hadoop-aliyun + compile + ${project.version} + org.apache.hadoop hadoop-sls diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index db002f46fc7..e7e876bc82a 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -47,6 +47,7 @@ hadoop-aws hadoop-kafka hadoop-azure-datalake + hadoop-aliyun diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java index 8ef881bbea9..bb9bca28a2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.api.records; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; @@ -38,7 +37,7 @@ public abstract class ReservationDefinition { @Unstable public static ReservationDefinition newInstance(long arrival, long deadline, ReservationRequests reservationRequests, String name, - String recurrenceExpression) { + String recurrenceExpression, Priority priority) { ReservationDefinition rDefinition = Records.newRecord(ReservationDefinition.class); rDefinition.setArrival(arrival); @@ -46,6 +45,7 @@ public abstract class ReservationDefinition { rDefinition.setReservationRequests(reservationRequests); rDefinition.setReservationName(name); rDefinition.setRecurrenceExpression(recurrenceExpression); + rDefinition.setPriority(priority); return rDefinition; } @@ -53,8 +53,8 @@ public abstract class ReservationDefinition { @Unstable public static ReservationDefinition newInstance(long arrival, long deadline, ReservationRequests reservationRequests, String name) { - ReservationDefinition rDefinition = - newInstance(arrival, deadline, reservationRequests, name, "0"); + ReservationDefinition rDefinition = newInstance(arrival, deadline, + reservationRequests, name, "0", Priority.UNDEFINED); return rDefinition; } @@ -130,7 +130,7 @@ public abstract class ReservationDefinition { * allocation in the scheduler */ @Public - @Evolving + @Unstable public abstract String getReservationName(); /** @@ -142,7 +142,7 @@ public abstract class ReservationDefinition { * allocation in the scheduler */ @Public - @Evolving + @Unstable public abstract void setReservationName(String name); /** @@ -160,7 +160,7 @@ public abstract class ReservationDefinition { * @return recurrence of this reservation */ @Public - @Evolving + @Unstable public abstract String getRecurrenceExpression(); /** @@ -178,7 +178,35 @@ public abstract class ReservationDefinition { * @param recurrenceExpression recurrence interval of this reservation */ @Public - @Evolving + @Unstable public abstract void setRecurrenceExpression(String recurrenceExpression); + /** + * Get the priority for this reservation. A lower number for priority + * indicates a higher priority reservation. Recurring reservations are + * always higher priority than non-recurring reservations. Priority for + * non-recurring reservations are only compared with non-recurring + * reservations. Likewise for recurring reservations. + * + * @return int representing the priority of the reserved resource + * allocation in the scheduler + */ + @Public + @Unstable + public abstract Priority getPriority(); + + /** + * Set the priority for this reservation. A lower number for priority + * indicates a higher priority reservation. Recurring reservations are + * always higher priority than non-recurring reservations. Priority for + * non-recurring reservations are only compared with non-recurring + * reservations. Likewise for recurring reservations. + * + * @param priority representing the priority of the reserved resource + * allocation in the scheduler + */ + @Public + @Unstable + public abstract void setPriority(Priority priority); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/URL.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/URL.java index aa28585ab17..19bfc32f442 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/URL.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/URL.java @@ -18,11 +18,15 @@ package org.apache.hadoop.yarn.api.records; +import com.google.common.annotations.VisibleForTesting; + import java.net.URI; import java.net.URISyntaxException; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.Records; @@ -52,7 +56,7 @@ public abstract class URL { @Public @Stable public abstract String getScheme(); - + /** * Set the scheme of the URL * @param scheme scheme of the URL @@ -68,7 +72,7 @@ public abstract class URL { @Public @Stable public abstract String getUserInfo(); - + /** * Set the user info of the URL. * @param userInfo user info of the URL @@ -84,7 +88,7 @@ public abstract class URL { @Public @Stable public abstract String getHost(); - + /** * Set the host of the URL. * @param host host of the URL @@ -100,7 +104,7 @@ public abstract class URL { @Public @Stable public abstract int getPort(); - + /** * Set the port of the URL * @param port port of the URL @@ -116,7 +120,7 @@ public abstract class URL { @Public @Stable public abstract String getFile(); - + /** * Set the file of the URL. * @param file file of the URL @@ -124,32 +128,20 @@ public abstract class URL { @Public @Stable public abstract void setFile(String file); - + @Public @Stable public Path toPath() throws URISyntaxException { - String scheme = getScheme() == null ? "" : getScheme(); - - String authority = ""; - if (getHost() != null) { - authority = getHost(); - if (getUserInfo() != null) { - authority = getUserInfo() + "@" + authority; - } - if (getPort() > 0) { - authority += ":" + getPort(); - } - } - - return new Path( - (new URI(scheme, authority, getFile(), null, null)).normalize()); + return new Path(new URI(getScheme(), getUserInfo(), + getHost(), getPort(), getFile(), null, null)); } - - @Public - @Stable - public static URL fromURI(URI uri) { + + + @Private + @VisibleForTesting + public static URL fromURI(URI uri, Configuration conf) { URL url = - RecordFactoryProvider.getRecordFactory(null).newRecordInstance( + RecordFactoryProvider.getRecordFactory(conf).newRecordInstance( URL.class); if (uri.getHost() != null) { url.setHost(uri.getHost()); @@ -162,7 +154,19 @@ public abstract class URL { url.setFile(uri.getPath()); return url; } - + + @Public + @Stable + public static URL fromURI(URI uri) { + return fromURI(uri, null); + } + + @Private + @VisibleForTesting + public static URL fromPath(Path path, Configuration conf) { + return fromURI(path.toUri(), conf); + } + @Public @Stable public static URL fromPath(Path path) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 4d433572683..3bd0dcc56db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -719,17 +719,29 @@ public class YarnConfiguration extends Configuration { + "leveldb-state-store.compaction-interval-secs"; public static final long DEFAULT_RM_LEVELDB_COMPACTION_INTERVAL_SECS = 3600; - /** The maximum number of completed applications RM keeps. */ + /** + * The maximum number of completed applications RM keeps. By default equals + * to {@link #DEFAULT_RM_MAX_COMPLETED_APPLICATIONS}. + */ public static final String RM_MAX_COMPLETED_APPLICATIONS = RM_PREFIX + "max-completed-applications"; - public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 10000; + public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 1000; /** - * The maximum number of completed applications RM state store keeps, by - * default equals to DEFAULT_RM_MAX_COMPLETED_APPLICATIONS + * The maximum number of completed applications RM state store keeps. By + * default equals to value of {@link #RM_MAX_COMPLETED_APPLICATIONS}. */ public static final String RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS = RM_PREFIX + "state-store.max-completed-applications"; + /** + * The default value for + * {@code yarn.resourcemanager.state-store.max-completed-applications}. + * @deprecated This default value is ignored and will be removed in a future + * release. The default value of + * {@code yarn.resourcemanager.state-store.max-completed-applications} is the + * value of {@link #RM_MAX_COMPLETED_APPLICATIONS}. + */ + @Deprecated public static final int DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS = DEFAULT_RM_MAX_COMPLETED_APPLICATIONS; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReplaceLabelsOnNodeRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReplaceLabelsOnNodeRequest.java index 28e261a5895..1b8e687b3dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReplaceLabelsOnNodeRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReplaceLabelsOnNodeRequest.java @@ -44,4 +44,12 @@ public abstract class ReplaceLabelsOnNodeRequest { @Public @Evolving public abstract Map> getNodeToLabels(); + + @Public + @Evolving + public abstract void setFailOnUnknownNodes(boolean failOnUnknownNodes); + + @Public + @Evolving + public abstract boolean getFailOnUnknownNodes(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index b9f30db46ee..16d80974c83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -99,10 +99,10 @@ message RemoveFromClusterNodeLabelsResponseProto { message ReplaceLabelsOnNodeRequestProto { repeated NodeIdToLabelsNameProto nodeToLabels = 1; + optional bool failOnUnknownNodes = 2; } message ReplaceLabelsOnNodeResponseProto { - } message UpdateNodeLabelsResponseProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index f7882953fe4..9c746fde303 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -489,6 +489,7 @@ message ReservationDefinitionProto { optional int64 deadline = 3; optional string reservation_name = 4; optional string recurrence_expression = 5 [default = "0"]; + optional PriorityProto priority = 6; } message ResourceAllocationRequestProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/TestURL.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/TestURL.java new file mode 100644 index 00000000000..b464eca5e59 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/TestURL.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.junit.Test; + +/** Test for the URL class. */ +public class TestURL { + + @Test + public void testConversion() throws Exception { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.IPC_RECORD_FACTORY_CLASS, + RecordFactoryForTest.class.getName()); + String[] pathStrs = new String[] {"/", ".", "foo/bar", "foo", + "/foo/bar/baz", "moo://bar/baz", "moo://bar:123/baz", "moo:///foo", + "moo://foo@bar:123/baz/foo", "moo://foo@bar/baz/foo", "moo://foo@bar", + "moo://foo:123"}; + for (String s : pathStrs) { + Path path = new Path(s); + assertEquals(path, URL.fromPath(path, conf).toPath()); + } + + Path p = new Path("/foo/bar#baz"); + assertEquals(p, URL.fromPath(p, conf).toPath()); + } + + /** Record factory that instantiates URLs for this test. */ + public static class RecordFactoryForTest implements RecordFactory { + private static final RecordFactoryForTest SELF = + new RecordFactoryForTest(); + @SuppressWarnings("unchecked") + @Override + public T newRecordInstance(Class clazz) { + return (T) new URLForTest(); + } + public static RecordFactory get() { + return SELF; + } + } + + /** URL fake for this test; sidesteps proto-URL dependency. */ + public static class URLForTest extends URL { + private String scheme, userInfo, host, file; + private int port; + public String getScheme() { + return scheme; + } + public void setScheme(String scheme) { + this.scheme = scheme; + } + public String getUserInfo() { + return userInfo; + } + public void setUserInfo(String userInfo) { + this.userInfo = userInfo; + } + public String getHost() { + return host; + } + public void setHost(String host) { + this.host = host; + } + public String getFile() { + return file; + } + public void setFile(String file) { + this.file = file; + } + public int getPort() { + return port; + } + public void setPort(int port) { + this.port = port; + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java index 6f6bb85829b..32216610612 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java @@ -455,10 +455,12 @@ public class AMRMClientImpl extends AMRMClient { protected void populateNMTokens(List nmTokens) { for (NMToken token : nmTokens) { String nodeId = token.getNodeId().toString(); - if (getNMTokenCache().containsToken(nodeId)) { - LOG.info("Replacing token for : " + nodeId); - } else { - LOG.info("Received new token for : " + nodeId); + if (LOG.isDebugEnabled()) { + if (getNMTokenCache().containsToken(nodeId)) { + LOG.debug("Replacing token for : " + nodeId); + } else { + LOG.debug("Received new token for : " + nodeId); + } } getNMTokenCache().setToken(nodeId, token.getToken()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java index b2bce22584e..c619e8ad54c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java @@ -78,8 +78,11 @@ public class ContainerManagementProtocolProxy { YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES + " (" + maxConnectedNMs + ") can not be less than 0."); } - LOG.info(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES + " : " - + maxConnectedNMs); + + if (LOG.isDebugEnabled()) { + LOG.debug(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES + " : " + + maxConnectedNMs); + } if (maxConnectedNMs > 0) { cmProxy = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index 7a898a12a38..640f8e36cfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -130,11 +130,13 @@ public class RMAdminCLI extends HAAdmin { new UsageInfo(" (label splitted by \",\")", "remove from cluster node labels")) .put("-replaceLabelsOnNode", - new UsageInfo( + new UsageInfo("[-failOnUnknownNodes] " + "<\"node1[:port]=label1,label2 node2[:port]=label1,label2\">", - "replace labels on nodes" - + " (please note that we do not support specifying multiple" - + " labels on a single host for now.)")) + "replace labels on nodes" + + " (please note that we do not support specifying multiple" + + " labels on a single host for now.)\n\t\t" + + "[-failOnUnknownNodes] is optional, when we set this" + + " option, it will fail if specified nodes are unknown.")) .put("-directlyAccessNodeLabelStore", new UsageInfo("", "This is DEPRECATED, will be removed in future releases. Directly access node label store, " + "with this option, all node label related operations" @@ -246,8 +248,8 @@ public class RMAdminCLI extends HAAdmin { " [-addToClusterNodeLabels <\"label1(exclusive=true)," + "label2(exclusive=false),label3\">]" + " [-removeFromClusterNodeLabels ]" + - " [-replaceLabelsOnNode <\"node1[:port]=label1,label2" + - " node2[:port]=label1\">]" + + " [-replaceLabelsOnNode [-failOnUnknownNodes] " + + "<\"node1[:port]=label1,label2 node2[:port]=label1\">]" + " [-directlyAccessNodeLabelStore]" + " [-refreshClusterMaxPriority]" + " [-updateNodeResource [NodeID] [MemSize] [vCores]" + @@ -302,7 +304,7 @@ public class RMAdminCLI extends HAAdmin { return ClientRMProxy.createRMProxy(conf, ResourceManagerAdministrationProtocol.class); } - + private int refreshQueues() throws IOException, YarnException { // Refresh the queue properties ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); @@ -657,14 +659,14 @@ public class RMAdminCLI extends HAAdmin { return map; } - private int replaceLabelsOnNodes(String args) throws IOException, - YarnException { + private int replaceLabelsOnNodes(String args, boolean failOnUnknownNodes) + throws IOException, YarnException { Map> map = buildNodeLabelsMapFromStr(args); - return replaceLabelsOnNodes(map); + return replaceLabelsOnNodes(map, failOnUnknownNodes); } - private int replaceLabelsOnNodes(Map> map) - throws IOException, YarnException { + private int replaceLabelsOnNodes(Map> map, + boolean failOnUnknownNodes) throws IOException, YarnException { if (directlyAccessNodeLabelStore) { getNodeLabelManagerInstance(getConf()).replaceLabelsOnNode(map); } else { @@ -672,11 +674,12 @@ public class RMAdminCLI extends HAAdmin { createAdminProtocol(); ReplaceLabelsOnNodeRequest request = ReplaceLabelsOnNodeRequest.newInstance(map); + request.setFailOnUnknownNodes(failOnUnknownNodes); adminProtocol.replaceLabelsOnNode(request); } return 0; } - + @Override public int run(String[] args) throws Exception { // -directlyAccessNodeLabelStore is a additional option for node label @@ -783,8 +786,16 @@ public class RMAdminCLI extends HAAdmin { System.err.println(NO_MAPPING_ERR_MSG); printUsage("", isHAEnabled); exitCode = -1; + } else if ("-failOnUnknownNodes".equals(args[i])) { + if (i + 1 >= args.length) { + System.err.println(NO_MAPPING_ERR_MSG); + printUsage("", isHAEnabled); + exitCode = -1; + } else { + exitCode = replaceLabelsOnNodes(args[i + 1], true); + } } else { - exitCode = replaceLabelsOnNodes(args[i]); + exitCode = replaceLabelsOnNodes(args[i], false); } } else { exitCode = -1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java new file mode 100644 index 00000000000..b9b4b02a14b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java @@ -0,0 +1,398 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api.impl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.NMToken; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.hadoop.yarn.client.api.NMTokenCache; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.Records; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import static org.junit.Assert.assertEquals; + +/** + * Class that tests the allocation of OPPORTUNISTIC containers through the + * centralized ResourceManager. + */ +public class TestOpportunisticContainerAllocation { + private static Configuration conf = null; + private static MiniYARNCluster yarnCluster = null; + private static YarnClient yarnClient = null; + private static List nodeReports = null; + private static ApplicationAttemptId attemptId = null; + private static int nodeCount = 3; + + private static final int ROLLING_INTERVAL_SEC = 13; + private static final long AM_EXPIRE_MS = 4000; + + private static Resource capability; + private static Priority priority; + private static Priority priority2; + private static String node; + private static String rack; + private static String[] nodes; + private static String[] racks; + private final static int DEFAULT_ITERATION = 3; + + @BeforeClass + public static void setup() throws Exception { + // start minicluster + conf = new YarnConfiguration(); + conf.setLong( + YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, + ROLLING_INTERVAL_SEC); + conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, AM_EXPIRE_MS); + conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100); + // set the minimum allocation so that resource decrease can go under 1024 + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); + conf.setBoolean( + YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true); + conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1); + yarnCluster = + new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1); + yarnCluster.init(conf); + yarnCluster.start(); + + // start rm client + yarnClient = YarnClient.createYarnClient(); + yarnClient.init(conf); + yarnClient.start(); + + // get node info + nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); + + priority = Priority.newInstance(1); + priority2 = Priority.newInstance(2); + capability = Resource.newInstance(1024, 1); + + node = nodeReports.get(0).getNodeId().getHost(); + rack = nodeReports.get(0).getRackName(); + nodes = new String[]{node}; + racks = new String[]{rack}; + } + + @Before + public void startApp() throws Exception { + // submit new app + ApplicationSubmissionContext appContext = + yarnClient.createApplication().getApplicationSubmissionContext(); + ApplicationId appId = appContext.getApplicationId(); + // set the application name + appContext.setApplicationName("Test"); + // Set the priority for the application master + Priority pri = Records.newRecord(Priority.class); + pri.setPriority(0); + appContext.setPriority(pri); + // Set the queue to which this application is to be submitted in the RM + appContext.setQueue("default"); + // Set up the container launch context for the application master + ContainerLaunchContext amContainer = BuilderUtils.newContainerLaunchContext( + Collections.emptyMap(), + new HashMap(), Arrays.asList("sleep", "100"), + new HashMap(), null, + new HashMap()); + appContext.setAMContainerSpec(amContainer); + appContext.setResource(Resource.newInstance(1024, 1)); + // Create the request to send to the applications manager + SubmitApplicationRequest appRequest = + Records.newRecord(SubmitApplicationRequest.class); + appRequest.setApplicationSubmissionContext(appContext); + // Submit the application to the applications manager + yarnClient.submitApplication(appContext); + + // wait for app to start + RMAppAttempt appAttempt = null; + while (true) { + ApplicationReport appReport = yarnClient.getApplicationReport(appId); + if (appReport.getYarnApplicationState() == + YarnApplicationState.ACCEPTED) { + attemptId = appReport.getCurrentApplicationAttemptId(); + appAttempt = yarnCluster.getResourceManager().getRMContext().getRMApps() + .get(attemptId.getApplicationId()).getCurrentAppAttempt(); + while (true) { + if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) { + break; + } + } + break; + } + } + // Just dig into the ResourceManager and get the AMRMToken just for the sake + // of testing. + UserGroupInformation.setLoginUser(UserGroupInformation + .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); + + // emulate RM setup of AMRM token in credentials by adding the token + // *before* setting the token service + UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken()); + appAttempt.getAMRMToken() + .setService(ClientRMProxy.getAMRMTokenService(conf)); + } + + @After + public void cancelApp() throws YarnException, IOException { + yarnClient.killApplication(attemptId.getApplicationId()); + attemptId = null; + } + + @AfterClass + public static void tearDown() { + if (yarnClient != null && + yarnClient.getServiceState() == Service.STATE.STARTED) { + yarnClient.stop(); + } + if (yarnCluster != null && + yarnCluster.getServiceState() == Service.STATE.STARTED) { + yarnCluster.stop(); + } + } + + @Test(timeout = 60000) + public void testAMRMClient() throws YarnException, IOException { + AMRMClient amClient = null; + try { + // start am rm client + amClient = AMRMClient.createAMRMClient(); + + //setting an instance NMTokenCache + amClient.setNMTokenCache(new NMTokenCache()); + //asserting we are not using the singleton instance cache + Assert.assertNotSame(NMTokenCache.getSingleton(), + amClient.getNMTokenCache()); + + amClient.init(conf); + amClient.start(); + + amClient.registerApplicationMaster("Host", 10000, ""); + + testAllocation((AMRMClientImpl)amClient); + + amClient + .unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, + null); + + } finally { + if (amClient != null && + amClient.getServiceState() == Service.STATE.STARTED) { + amClient.stop(); + } + } + } + + private void testAllocation( + final AMRMClientImpl amClient) + throws YarnException, IOException { + // setup container request + + assertEquals(0, amClient.ask.size()); + assertEquals(0, amClient.release.size()); + + amClient.addContainerRequest( + new AMRMClient.ContainerRequest(capability, nodes, racks, priority)); + amClient.addContainerRequest( + new AMRMClient.ContainerRequest(capability, nodes, racks, priority)); + amClient.addContainerRequest( + new AMRMClient.ContainerRequest(capability, nodes, racks, priority)); + amClient.addContainerRequest( + new AMRMClient.ContainerRequest(capability, nodes, racks, priority)); + amClient.addContainerRequest( + new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, + true, null, + ExecutionTypeRequest.newInstance( + ExecutionType.OPPORTUNISTIC, true))); + amClient.addContainerRequest( + new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, + true, null, + ExecutionTypeRequest.newInstance( + ExecutionType.OPPORTUNISTIC, true))); + + amClient.removeContainerRequest( + new AMRMClient.ContainerRequest(capability, nodes, racks, priority)); + amClient.removeContainerRequest( + new AMRMClient.ContainerRequest(capability, nodes, racks, priority)); + amClient.removeContainerRequest( + new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, + true, null, + ExecutionTypeRequest.newInstance( + ExecutionType.OPPORTUNISTIC, true))); + + int containersRequestedNode = amClient.getTable(0).get(priority, + node, ExecutionType.GUARANTEED, capability).remoteRequest + .getNumContainers(); + int containersRequestedRack = amClient.getTable(0).get(priority, + rack, ExecutionType.GUARANTEED, capability).remoteRequest + .getNumContainers(); + int containersRequestedAny = amClient.getTable(0).get(priority, + ResourceRequest.ANY, ExecutionType.GUARANTEED, capability) + .remoteRequest.getNumContainers(); + int oppContainersRequestedAny = + amClient.getTable(0).get(priority2, ResourceRequest.ANY, + ExecutionType.OPPORTUNISTIC, capability).remoteRequest + .getNumContainers(); + + assertEquals(2, containersRequestedNode); + assertEquals(2, containersRequestedRack); + assertEquals(2, containersRequestedAny); + assertEquals(1, oppContainersRequestedAny); + + assertEquals(4, amClient.ask.size()); + assertEquals(0, amClient.release.size()); + + // RM should allocate container within 2 calls to allocate() + int allocatedContainerCount = 0; + int allocatedOpportContainerCount = 0; + int iterationsLeft = 10; + Set releases = new TreeSet<>(); + + amClient.getNMTokenCache().clearCache(); + Assert.assertEquals(0, + amClient.getNMTokenCache().numberOfTokensInCache()); + HashMap receivedNMTokens = new HashMap<>(); + + while (allocatedContainerCount < + containersRequestedAny + oppContainersRequestedAny + && iterationsLeft-- > 0) { + AllocateResponse allocResponse = amClient.allocate(0.1f); + assertEquals(0, amClient.ask.size()); + assertEquals(0, amClient.release.size()); + + allocatedContainerCount += allocResponse.getAllocatedContainers() + .size(); + for (Container container : allocResponse.getAllocatedContainers()) { + if (container.getExecutionType() == ExecutionType.OPPORTUNISTIC) { + allocatedOpportContainerCount++; + } + ContainerId rejectContainerId = container.getId(); + releases.add(rejectContainerId); + } + + for (NMToken token : allocResponse.getNMTokens()) { + String nodeID = token.getNodeId().toString(); + receivedNMTokens.put(nodeID, token.getToken()); + } + + if (allocatedContainerCount < containersRequestedAny) { + // sleep to let NM's heartbeat to RM and trigger allocations + sleep(100); + } + } + + assertEquals(allocatedContainerCount, + containersRequestedAny + oppContainersRequestedAny); + assertEquals(allocatedOpportContainerCount, oppContainersRequestedAny); + for (ContainerId rejectContainerId : releases) { + amClient.releaseAssignedContainer(rejectContainerId); + } + assertEquals(3, amClient.release.size()); + assertEquals(0, amClient.ask.size()); + + // need to tell the AMRMClient that we don't need these resources anymore + amClient.removeContainerRequest( + new AMRMClient.ContainerRequest(capability, nodes, racks, priority)); + amClient.removeContainerRequest( + new AMRMClient.ContainerRequest(capability, nodes, racks, priority)); + amClient.removeContainerRequest( + new AMRMClient.ContainerRequest(capability, nodes, racks, priority2, 0, + true, null, + ExecutionTypeRequest.newInstance( + ExecutionType.OPPORTUNISTIC, true))); + assertEquals(4, amClient.ask.size()); + + iterationsLeft = 3; + // do a few iterations to ensure RM is not going to send new containers + while (iterationsLeft-- > 0) { + // inform RM of rejection + AllocateResponse allocResponse = amClient.allocate(0.1f); + // RM did not send new containers because AM does not need any + assertEquals(0, allocResponse.getAllocatedContainers().size()); + if (allocResponse.getCompletedContainersStatuses().size() > 0) { + for (ContainerStatus cStatus : allocResponse + .getCompletedContainersStatuses()) { + if (releases.contains(cStatus.getContainerId())) { + assertEquals(cStatus.getState(), ContainerState.COMPLETE); + assertEquals(-100, cStatus.getExitStatus()); + releases.remove(cStatus.getContainerId()); + } + } + } + if (iterationsLeft > 0) { + // sleep to make sure NM's heartbeat + sleep(100); + } + } + assertEquals(0, amClient.ask.size()); + assertEquals(0, amClient.release.size()); + } + + private void sleep(int sleepTime) { + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java index bea6e39e1dd..9e20a4359a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java @@ -469,7 +469,7 @@ public class TestRMAdminCLI { "[username]] [-addToClusterNodeLabels " + "<\"label1(exclusive=true),label2(exclusive=false),label3\">] " + "[-removeFromClusterNodeLabels ] " + - "[-replaceLabelsOnNode " + + "[-replaceLabelsOnNode [-failOnUnknownNodes] " + "<\"node1[:port]=label1,label2 node2[:port]=label1\">] " + "[-directlyAccessNodeLabelStore] [-refreshClusterMaxPriority] " + "[-updateNodeResource [NodeID] [MemSize] [vCores] " + @@ -564,6 +564,7 @@ public class TestRMAdminCLI { + " [username]] [-addToClusterNodeLabels <\"label1(exclusive=true)," + "label2(exclusive=false),label3\">]" + " [-removeFromClusterNodeLabels ] [-replaceLabelsOnNode " + + "[-failOnUnknownNodes] " + "<\"node1[:port]=label1,label2 node2[:port]=label1\">] [-directlyAccessNodeLabelStore] " + "[-refreshClusterMaxPriority] " + "[-updateNodeResource [NodeID] [MemSize] [vCores] " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java index b30cd2a2b90..49aef11976e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java @@ -18,8 +18,10 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationRequests; +import org.apache.hadoop.yarn.proto.YarnProtos; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationDefinitionProto; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationDefinitionProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestsProto; @@ -32,6 +34,7 @@ public class ReservationDefinitionPBImpl extends ReservationDefinition { boolean viaProto = false; private ReservationRequests reservationReqs; + private Priority priority = null; public ReservationDefinitionPBImpl() { builder = ReservationDefinitionProto.newBuilder(); @@ -150,6 +153,33 @@ public class ReservationDefinitionPBImpl extends ReservationDefinition { builder.setReservationName(name); } + @Override + public Priority getPriority() { + ReservationDefinitionProtoOrBuilder p = viaProto ? proto : builder; + if (this.priority != null) { + return this.priority; + } + if (!p.hasPriority()) { + return Priority.UNDEFINED; + } + this.priority = convertFromProtoFormat(p.getPriority()); + return this.priority; + } + + @Override + public void setPriority(Priority priority) { + maybeInitBuilder(); + if (priority == null) { + this.priority = Priority.UNDEFINED; + } + this.priority = priority; + } + + private PriorityPBImpl convertFromProtoFormat( + YarnProtos.PriorityProto p) { + return new PriorityPBImpl(p); + } + private ReservationRequestsPBImpl convertFromProtoFormat( ReservationRequestsProto p) { return new ReservationRequestsPBImpl(p); @@ -164,6 +194,7 @@ public class ReservationDefinitionPBImpl extends ReservationDefinition { return "{Arrival: " + getArrival() + ", Deadline: " + getDeadline() + ", Reservation Name: " + getReservationName() + ", Recurrence expression: " + getRecurrenceExpression() + + ", Priority: " + getPriority().toString() + ", Resources: " + getReservationRequests() + "}"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java index 22e561cd94a..3b15b27d78b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java @@ -146,10 +146,22 @@ public class ReplaceLabelsOnNodeRequestPBImpl extends nodeIdToLabels.putAll(map); } + @Override + public boolean getFailOnUnknownNodes() { + ReplaceLabelsOnNodeRequestProtoOrBuilder p = viaProto ? proto : builder; + return p.getFailOnUnknownNodes(); + } + + @Override + public void setFailOnUnknownNodes(boolean failOnUnknownNodes) { + maybeInitBuilder(); + builder.setFailOnUnknownNodes(failOnUnknownNodes); + } + private NodeIdProto convertToProtoFormat(NodeId t) { return ((NodeIdPBImpl) t).getProto(); } - + @Override public int hashCode() { assert false : "hashCode not designed"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index 80d49c3771b..29bc277829b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -406,15 +406,14 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { continue; } - total += - Math.min(info.sharedDirty, info.pss) + info.privateDirty - + info.privateClean; + // Account for anonymous to know the amount of + // memory reclaimable by killing the process + total += info.anonymous; + if (LOG.isDebugEnabled()) { LOG.debug(" total(" + olderThanAge + "): PID : " + p.getPid() - + ", SharedDirty : " + info.sharedDirty + ", PSS : " - + info.pss + ", Private_Dirty : " + info.privateDirty - + ", Private_Clean : " + info.privateClean + ", total : " - + (total * KB_TO_BYTES)); + + ", info : " + info.toString() + + ", total : " + (total * KB_TO_BYTES)); } } } @@ -877,6 +876,7 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { private int sharedDirty; private int privateClean; private int privateDirty; + private int anonymous; private int referenced; private String regionName; private String permission; @@ -929,6 +929,10 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { return referenced; } + public int getAnonymous() { + return anonymous; + } + public void setMemInfo(String key, String value) { MemInfo info = MemInfo.getMemInfoByName(key); int val = 0; @@ -969,6 +973,9 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { case REFERENCED: referenced = val; break; + case ANONYMOUS: + anonymous = val; + break; default: break; } @@ -999,10 +1006,7 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { .append(MemInfo.REFERENCED.name + ":" + this.getReferenced()) .append(" kB\n"); sb.append("\t") - .append(MemInfo.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty()) - .append(" kB\n"); - sb.append("\t") - .append(MemInfo.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty()) + .append(MemInfo.ANONYMOUS.name + ":" + this.getAnonymous()) .append(" kB\n"); return sb.toString(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java index a34273c9f47..ee9100f8e78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java @@ -41,4 +41,5 @@ public interface YarnWebParams { String NODE_LABEL = "node.label"; String WEB_UI_TYPE = "web.ui.type"; String NEXT_REFRESH_INTERVAL = "next.refresh.interval"; + String ERROR_MESSAGE = "error.message"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 524afecd852..f37c689ba6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -417,7 +417,7 @@ the applications remembered in RM memory. Any values larger than ${yarn.resourcemanager.max-completed-applications} will be reset to ${yarn.resourcemanager.max-completed-applications}. - Note that this value impacts the RM recovery performance.Typically, + Note that this value impacts the RM recovery performance. Typically, a smaller value indicates better performance on RM recovery. yarn.resourcemanager.state-store.max-completed-applications @@ -687,7 +687,7 @@ The maximum number of completed applications RM keeps. yarn.resourcemanager.max-completed-applications - 10000 + 1000 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java new file mode 100644 index 00000000000..21a737dd557 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java @@ -0,0 +1,264 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Assert; + +import java.lang.reflect.*; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * Generic helper class to validate protocol records. + */ +public class BasePBImplRecordsTest { + static final Log LOG = LogFactory.getLog(BasePBImplRecordsTest.class); + + @SuppressWarnings("checkstyle:visibilitymodifier") + protected static HashMap typeValueCache = + new HashMap(); + private static Random rand = new Random(); + private static byte [] bytes = new byte[] {'1', '2', '3', '4'}; + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static Object genTypeValue(Type type) { + Object ret = typeValueCache.get(type); + if (ret != null) { + return ret; + } + // only use positive primitive values + if (type.equals(boolean.class)) { + return rand.nextBoolean(); + } else if (type.equals(byte.class)) { + return bytes[rand.nextInt(4)]; + } else if (type.equals(int.class) || type.equals(Integer.class)) { + return rand.nextInt(1000000); + } else if (type.equals(long.class) || type.equals(Long.class)) { + return Long.valueOf(rand.nextInt(1000000)); + } else if (type.equals(float.class)) { + return rand.nextFloat(); + } else if (type.equals(double.class)) { + return rand.nextDouble(); + } else if (type.equals(String.class)) { + return String.format("%c%c%c", + 'a' + rand.nextInt(26), + 'a' + rand.nextInt(26), + 'a' + rand.nextInt(26)); + } else if (type instanceof Class) { + Class clazz = (Class)type; + if (clazz.isArray()) { + Class compClass = clazz.getComponentType(); + if (compClass != null) { + ret = Array.newInstance(compClass, 2); + Array.set(ret, 0, genTypeValue(compClass)); + Array.set(ret, 1, genTypeValue(compClass)); + } + } else if (clazz.isEnum()) { + Object [] values = clazz.getEnumConstants(); + ret = values[rand.nextInt(values.length)]; + } else if (clazz.equals(ByteBuffer.class)) { + // return new ByteBuffer every time + // to prevent potential side effects + ByteBuffer buff = ByteBuffer.allocate(4); + rand.nextBytes(buff.array()); + return buff; + } + } else if (type instanceof ParameterizedType) { + ParameterizedType pt = (ParameterizedType)type; + Type rawType = pt.getRawType(); + Type [] params = pt.getActualTypeArguments(); + // only support EnumSet, List, Set, Map + if (rawType.equals(EnumSet.class)) { + if (params[0] instanceof Class) { + Class c = (Class)(params[0]); + return EnumSet.allOf(c); + } + } if (rawType.equals(List.class)) { + ret = Lists.newArrayList(genTypeValue(params[0])); + } else if (rawType.equals(Set.class)) { + ret = Sets.newHashSet(genTypeValue(params[0])); + } else if (rawType.equals(Map.class)) { + Map map = Maps.newHashMap(); + map.put(genTypeValue(params[0]), genTypeValue(params[1])); + ret = map; + } + } + if (ret == null) { + throw new IllegalArgumentException("type " + type + " is not supported"); + } + typeValueCache.put(type, ret); + return ret; + } + + /** + * this method generate record instance by calling newIntance + * using reflection, add register the generated value to typeValueCache + */ + @SuppressWarnings("rawtypes") + protected static Object generateByNewInstance(Class clazz) throws Exception { + Object ret = typeValueCache.get(clazz); + if (ret != null) { + return ret; + } + Method newInstance = null; + Type [] paramTypes = new Type[0]; + // get newInstance method with most parameters + for (Method m : clazz.getMethods()) { + int mod = m.getModifiers(); + if (m.getDeclaringClass().equals(clazz) && + Modifier.isPublic(mod) && + Modifier.isStatic(mod) && + m.getName().equals("newInstance")) { + Type [] pts = m.getGenericParameterTypes(); + if (newInstance == null + || (pts.length > paramTypes.length)) { + newInstance = m; + paramTypes = pts; + } + } + } + if (newInstance == null) { + throw new IllegalArgumentException("type " + clazz.getName() + + " does not have newInstance method"); + } + Object [] args = new Object[paramTypes.length]; + for (int i=0;i Map getGetSetPairs(Class recordClass) + throws Exception { + Map ret = new HashMap(); + Method [] methods = recordClass.getDeclaredMethods(); + // get all get methods + for (int i = 0; i < methods.length; i++) { + Method m = methods[i]; + int mod = m.getModifiers(); + if (m.getDeclaringClass().equals(recordClass) && + Modifier.isPublic(mod) && + (!Modifier.isStatic(mod))) { + String name = m.getName(); + if (name.equals("getProto")) { + continue; + } + if ((name.length() > 3) && name.startsWith("get") && + (m.getParameterTypes().length == 0)) { + String propertyName = name.substring(3); + Type valueType = m.getGenericReturnType(); + GetSetPair p = ret.get(propertyName); + if (p == null) { + p = new GetSetPair(); + p.propertyName = propertyName; + p.type = valueType; + p.getMethod = m; + ret.put(propertyName, p); + } else { + Assert.fail("Multiple get method with same name: " + recordClass + + p.propertyName); + } + } + } + } + // match get methods with set methods + for (int i = 0; i < methods.length; i++) { + Method m = methods[i]; + int mod = m.getModifiers(); + if (m.getDeclaringClass().equals(recordClass) && + Modifier.isPublic(mod) && + (!Modifier.isStatic(mod))) { + String name = m.getName(); + if (name.startsWith("set") && (m.getParameterTypes().length == 1)) { + String propertyName = name.substring(3); + Type valueType = m.getGenericParameterTypes()[0]; + GetSetPair p = ret.get(propertyName); + if (p != null && p.type.equals(valueType)) { + p.setMethod = m; + } + } + } + } + // exclude incomplete get/set pair, and generate test value + Iterator> itr = ret.entrySet().iterator(); + while (itr.hasNext()) { + Map.Entry cur = itr.next(); + GetSetPair gsp = cur.getValue(); + if ((gsp.getMethod == null) || + (gsp.setMethod == null)) { + LOG.info(String.format("Exclude protential property: %s\n", gsp.propertyName)); + itr.remove(); + } else { + LOG.info(String.format("New property: %s type: %s", gsp.toString(), gsp.type)); + gsp.testValue = genTypeValue(gsp.type); + LOG.info(String.format(" testValue: %s\n", gsp.testValue)); + } + } + return ret; + } + + protected void validatePBImplRecord(Class recordClass, + Class

    protoClass) + throws Exception { + LOG.info(String.format("Validate %s %s\n", recordClass.getName(), + protoClass.getName())); + Constructor emptyConstructor = recordClass.getConstructor(); + Constructor pbConstructor = recordClass.getConstructor(protoClass); + Method getProto = recordClass.getDeclaredMethod("getProto"); + Map getSetPairs = getGetSetPairs(recordClass); + R origRecord = emptyConstructor.newInstance(); + for (GetSetPair gsp : getSetPairs.values()) { + gsp.setMethod.invoke(origRecord, gsp.testValue); + } + Object ret = getProto.invoke(origRecord); + Assert.assertNotNull(recordClass.getName() + "#getProto returns null", ret); + if (!(protoClass.isAssignableFrom(ret.getClass()))) { + Assert.fail("Illegal getProto method return type: " + ret.getClass()); + } + R deserRecord = pbConstructor.newInstance(ret); + Assert.assertEquals("whole " + recordClass + " records should be equal", + origRecord, deserRecord); + for (GetSetPair gsp : getSetPairs.values()) { + Object origValue = gsp.getMethod.invoke(origRecord); + Object deserValue = gsp.getMethod.invoke(deserRecord); + Assert.assertEquals("property " + recordClass.getName() + "#" + + gsp.propertyName + " should be equal", origValue, deserValue); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 527048650c4..4b71282ffe0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -16,26 +16,8 @@ * limitations under the License. */ package org.apache.hadoop.yarn.api; -import java.io.IOException; -import java.lang.reflect.Array; -import java.lang.reflect.Constructor; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.nio.ByteBuffer; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.Set; - +import com.google.common.collect.ImmutableSet; import org.apache.commons.lang.math.LongRange; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto; import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto; @@ -343,127 +325,12 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import java.io.IOException; -public class TestPBImplRecords { - static final Log LOG = LogFactory.getLog(TestPBImplRecords.class); - - private static HashMap typeValueCache = new HashMap(); - private static Random rand = new Random(); - private static byte [] bytes = new byte[] {'1', '2', '3', '4'}; - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static Object genTypeValue(Type type) { - Object ret = typeValueCache.get(type); - if (ret != null) { - return ret; - } - // only use positive primitive values - if (type.equals(boolean.class)) { - return rand.nextBoolean(); - } else if (type.equals(byte.class)) { - return bytes[rand.nextInt(4)]; - } else if (type.equals(int.class) || type.equals(Integer.class)) { - return rand.nextInt(1000000); - } else if (type.equals(long.class) || type.equals(Long.class)) { - return Long.valueOf(rand.nextInt(1000000)); - } else if (type.equals(float.class)) { - return rand.nextFloat(); - } else if (type.equals(double.class)) { - return rand.nextDouble(); - } else if (type.equals(String.class)) { - return String.format("%c%c%c", - 'a' + rand.nextInt(26), - 'a' + rand.nextInt(26), - 'a' + rand.nextInt(26)); - } else if (type instanceof Class) { - Class clazz = (Class)type; - if (clazz.isArray()) { - Class compClass = clazz.getComponentType(); - if (compClass != null) { - ret = Array.newInstance(compClass, 2); - Array.set(ret, 0, genTypeValue(compClass)); - Array.set(ret, 1, genTypeValue(compClass)); - } - } else if (clazz.isEnum()) { - Object [] values = clazz.getEnumConstants(); - ret = values[rand.nextInt(values.length)]; - } else if (clazz.equals(ByteBuffer.class)) { - // return new ByteBuffer every time - // to prevent potential side effects - ByteBuffer buff = ByteBuffer.allocate(4); - rand.nextBytes(buff.array()); - return buff; - } - } else if (type instanceof ParameterizedType) { - ParameterizedType pt = (ParameterizedType)type; - Type rawType = pt.getRawType(); - Type [] params = pt.getActualTypeArguments(); - // only support EnumSet, List, Set, Map - if (rawType.equals(EnumSet.class)) { - if (params[0] instanceof Class) { - Class c = (Class)(params[0]); - return EnumSet.allOf(c); - } - } if (rawType.equals(List.class)) { - ret = Lists.newArrayList(genTypeValue(params[0])); - } else if (rawType.equals(Set.class)) { - ret = Sets.newHashSet(genTypeValue(params[0])); - } else if (rawType.equals(Map.class)) { - Map map = Maps.newHashMap(); - map.put(genTypeValue(params[0]), genTypeValue(params[1])); - ret = map; - } - } - if (ret == null) { - throw new IllegalArgumentException("type " + type + " is not supported"); - } - typeValueCache.put(type, ret); - return ret; - } - - /** - * this method generate record instance by calling newIntance - * using reflection, add register the generated value to typeValueCache - */ - @SuppressWarnings("rawtypes") - private static Object generateByNewInstance(Class clazz) throws Exception { - Object ret = typeValueCache.get(clazz); - if (ret != null) { - return ret; - } - Method newInstance = null; - Type [] paramTypes = new Type[0]; - // get newInstance method with most parameters - for (Method m : clazz.getMethods()) { - int mod = m.getModifiers(); - if (m.getDeclaringClass().equals(clazz) && - Modifier.isPublic(mod) && - Modifier.isStatic(mod) && - m.getName().equals("newInstance")) { - Type [] pts = m.getGenericParameterTypes(); - if (newInstance == null - || (pts.length > paramTypes.length)) { - newInstance = m; - paramTypes = pts; - } - } - } - if (newInstance == null) { - throw new IllegalArgumentException("type " + clazz.getName() + - " does not have newInstance method"); - } - Object [] args = new Object[paramTypes.length]; - for (int i=0;i Map getGetSetPairs(Class recordClass) - throws Exception { - Map ret = new HashMap(); - Method [] methods = recordClass.getDeclaredMethods(); - // get all get methods - for (int i = 0; i < methods.length; i++) { - Method m = methods[i]; - int mod = m.getModifiers(); - if (m.getDeclaringClass().equals(recordClass) && - Modifier.isPublic(mod) && - (!Modifier.isStatic(mod))) { - String name = m.getName(); - if (name.equals("getProto")) { - continue; - } - if ((name.length() > 3) && name.startsWith("get") && - (m.getParameterTypes().length == 0)) { - String propertyName = name.substring(3); - Type valueType = m.getGenericReturnType(); - GetSetPair p = ret.get(propertyName); - if (p == null) { - p = new GetSetPair(); - p.propertyName = propertyName; - p.type = valueType; - p.getMethod = m; - ret.put(propertyName, p); - } else { - Assert.fail("Multiple get method with same name: " + recordClass - + p.propertyName); - } - } - } - } - // match get methods with set methods - for (int i = 0; i < methods.length; i++) { - Method m = methods[i]; - int mod = m.getModifiers(); - if (m.getDeclaringClass().equals(recordClass) && - Modifier.isPublic(mod) && - (!Modifier.isStatic(mod))) { - String name = m.getName(); - if (name.startsWith("set") && (m.getParameterTypes().length == 1)) { - String propertyName = name.substring(3); - Type valueType = m.getGenericParameterTypes()[0]; - GetSetPair p = ret.get(propertyName); - if (p != null && p.type.equals(valueType)) { - p.setMethod = m; - } - } - } - } - // exclude incomplete get/set pair, and generate test value - Iterator> itr = ret.entrySet().iterator(); - while (itr.hasNext()) { - Entry cur = itr.next(); - GetSetPair gsp = cur.getValue(); - if ((gsp.getMethod == null) || - (gsp.setMethod == null)) { - LOG.info(String.format("Exclude protential property: %s\n", gsp.propertyName)); - itr.remove(); - } else { - LOG.info(String.format("New property: %s type: %s", gsp.toString(), gsp.type)); - gsp.testValue = genTypeValue(gsp.type); - LOG.info(String.format(" testValue: %s\n", gsp.testValue)); - } - } - return ret; - } - - private void validatePBImplRecord(Class recordClass, - Class

    protoClass) - throws Exception { - LOG.info(String.format("Validate %s %s\n", recordClass.getName(), - protoClass.getName())); - Constructor emptyConstructor = recordClass.getConstructor(); - Constructor pbConstructor = recordClass.getConstructor(protoClass); - Method getProto = recordClass.getDeclaredMethod("getProto"); - Map getSetPairs = getGetSetPairs(recordClass); - R origRecord = emptyConstructor.newInstance(); - for (GetSetPair gsp : getSetPairs.values()) { - gsp.setMethod.invoke(origRecord, gsp.testValue); - } - Object ret = getProto.invoke(origRecord); - Assert.assertNotNull(recordClass.getName() + "#getProto returns null", ret); - if (!(protoClass.isAssignableFrom(ret.getClass()))) { - Assert.fail("Illegal getProto method return type: " + ret.getClass()); - } - R deserRecord = pbConstructor.newInstance(ret); - Assert.assertEquals("whole " + recordClass + " records should be equal", - origRecord, deserRecord); - for (GetSetPair gsp : getSetPairs.values()) { - Object origValue = gsp.getMethod.invoke(origRecord); - Object deserValue = gsp.getMethod.invoke(deserRecord); - Assert.assertEquals("property " + recordClass.getName() + "#" - + gsp.propertyName + " should be equal", origValue, deserValue); - } - } - @Test public void testAllocateRequestPBImpl() throws Exception { validatePBImplRecord(AllocateRequestPBImpl.class, AllocateRequestProto.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java index fa4e8c8f14c..841d333ce07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java @@ -369,21 +369,24 @@ public class TestProcfsBasedProcessTree { List memoryMappingList = procMemInfo[i].getMemoryInfoList(); memoryMappingList.add(constructMemoryMappingInfo( - "7f56c177c000-7f56c177d000 " + "7f56c177c000-7f56c177d000 " + "rw-p 00010000 08:02 40371558 " + "/grid/0/jdk1.7.0_25/jre/lib/amd64/libnio.so", - new String[] { "4", "4", "25", "4", "25", "15", "10", "4", "0", "0", - "0", "4", "4" })); + // Format: size, rss, pss, shared_clean, shared_dirty, private_clean + // private_dirty, referenced, anon, anon-huge-pages, swap, + // kernel_page_size, mmu_page_size + new String[] {"4", "4", "25", "4", "25", "15", "10", "4", "10", "0", + "0", "4", "4"})); memoryMappingList.add(constructMemoryMappingInfo( - "7fb09382e000-7fb09382f000 r--s 00003000 " + "08:02 25953545", - new String[] { "4", "4", "25", "4", "0", "15", "10", "4", "0", "0", - "0", "4", "4" })); + "7fb09382e000-7fb09382f000 r--s 00003000 " + "08:02 25953545", + new String[] {"4", "4", "25", "4", "0", "15", "10", "4", "10", "0", + "0", "4", "4"})); memoryMappingList.add(constructMemoryMappingInfo( - "7e8790000-7e8b80000 r-xs 00000000 00:00 0", new String[] { "4", "4", - "25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4" })); + "7e8790000-7e8b80000 r-xs 00000000 00:00 0", new String[] {"4", "4", + "25", "4", "0", "15", "10", "4", "10", "0", "0", "4", "4"})); memoryMappingList.add(constructMemoryMappingInfo( - "7da677000-7e0dcf000 rw-p 00000000 00:00 0", new String[] { "4", "4", - "25", "4", "50", "15", "10", "4", "0", "0", "0", "4", "4" })); + "7da677000-7e0dcf000 rw-p 00000000 00:00 0", new String[] {"4", "4", + "25", "4", "50", "15", "10", "4", "10", "0", "0", "4", "4"})); } } @@ -471,13 +474,12 @@ public class TestProcfsBasedProcessTree { // Check by enabling smaps setSmapsInProceTree(processTree, true); - // RSS=Min(shared_dirty,PSS)+PrivateClean+PrivateDirty (exclude r-xs, - // r--s) + // anon (exclude r-xs,r--s) Assert.assertEquals("rss memory does not match", - (100 * KB_TO_BYTES * 3), processTree.getRssMemorySize()); + (20 * KB_TO_BYTES * 3), processTree.getRssMemorySize()); // verify old API Assert.assertEquals("rss memory (old API) does not match", - (100 * KB_TO_BYTES * 3), processTree.getCumulativeRssmem()); + (20 * KB_TO_BYTES * 3), processTree.getCumulativeRssmem()); // test the cpu time again to see if it cumulates procInfos[0] = @@ -621,10 +623,10 @@ public class TestProcfsBasedProcessTree { cumuRssMem, processTree.getCumulativeRssmem()); } else { Assert.assertEquals("rssmem does not include new process", - 100 * KB_TO_BYTES * 4, processTree.getRssMemorySize()); + 20 * KB_TO_BYTES * 4, processTree.getRssMemorySize()); // verify old API Assert.assertEquals("rssmem (old API) does not include new process", - 100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem()); + 20 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem()); } // however processes older than 1 iteration will retain the older value @@ -650,11 +652,11 @@ public class TestProcfsBasedProcessTree { } else { Assert.assertEquals( "rssmem shouldn't have included new process", - 100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1)); + 20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1)); // Verify old API Assert.assertEquals( "rssmem (old API) shouldn't have included new process", - 100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(1)); + 20 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(1)); } // one more process @@ -696,11 +698,11 @@ public class TestProcfsBasedProcessTree { } else { Assert.assertEquals( "rssmem shouldn't have included new processes", - 100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2)); + 20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2)); // Verify old API Assert.assertEquals( "rssmem (old API) shouldn't have included new processes", - 100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(2)); + 20 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(2)); } // processes older than 1 iteration should not include new process, @@ -727,10 +729,10 @@ public class TestProcfsBasedProcessTree { } else { Assert.assertEquals( "rssmem shouldn't have included new processes", - 100 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1)); + 20 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1)); Assert.assertEquals( "rssmem (old API) shouldn't have included new processes", - 100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem(1)); + 20 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem(1)); } // no processes older than 3 iterations diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 84d45439445..feeafdd5d4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -351,6 +351,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService } } List events = entity.getEvents(); + long updatedTimeStamp = 0L; if (events != null) { for (TimelineEvent event : events) { if (event.getEventType().equals( @@ -358,9 +359,16 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService createdTime = event.getTimestamp(); } else if (event.getEventType().equals( ApplicationMetricsConstants.UPDATED_EVENT_TYPE)) { - // TODO: YARN-5101. This type of events are parsed in - // time-stamp descending order which means the previous event - // could override the information from the later same type of event. + // This type of events are parsed in time-stamp descending order + // which means the previous event could override the information + // from the later same type of event. Hence compare timestamp + // before over writing. + if (event.getTimestamp() > updatedTimeStamp) { + updatedTimeStamp = event.getTimestamp(); + } else { + continue; + } + Map eventInfo = event.getEventInfo(); if (eventInfo == null) { continue; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index b65b22b4776..dd1a4530eed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -551,23 +551,27 @@ public class TestApplicationHistoryManagerOnTimelineStore { entity.addEvent(tEvent); if (enableUpdateEvent) { tEvent = new TimelineEvent(); - createAppModifiedEvent(appId, tEvent, "changed queue", 5); + long updatedTimeIndex = 4L; + createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", + 5); entity.addEvent(tEvent); // Change priority alone tEvent = new TimelineEvent(); - createAppModifiedEvent(appId, tEvent, "changed queue", 6); + createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", + 6); // Now change queue tEvent = new TimelineEvent(); - createAppModifiedEvent(appId, tEvent, "changed queue1", 6); + createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, + "changed queue1", 6); entity.addEvent(tEvent); } return entity; } private static void createAppModifiedEvent(ApplicationId appId, - TimelineEvent tEvent, String queue, int priority) { + TimelineEvent tEvent, long updatedTimeIndex, String queue, int priority) { tEvent.setEventType(ApplicationMetricsConstants.UPDATED_EVENT_TYPE); - tEvent.setTimestamp(Integer.MAX_VALUE + 4L + appId.getId()); + tEvent.setTimestamp(Integer.MAX_VALUE + updatedTimeIndex + appId.getId()); Map eventInfo = new HashMap(); eventInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, queue); eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java index 9b2fd3857bd..9c158e95b82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.util.resource.Resources; import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -144,15 +145,6 @@ public class OpportunisticContainerAllocator { this.containerIdCounter.set(containerIdStart); } - /** - * Sets the underlying Atomic Long. To be used when implementation needs to - * share the underlying AtomicLong of an existing counter. - * @param counter AtomicLong - */ - public void setContainerIdCounter(AtomicLong counter) { - this.containerIdCounter = counter; - } - /** * Generates a new long value. Default implementation increments the * underlying AtomicLong. Sub classes are encouraged to over-ride this @@ -213,6 +205,10 @@ public class OpportunisticContainerAllocator { PartitionedResourceRequests partitionedAsks = partitionAskList(request.getAskList()); + if (partitionedAsks.getOpportunistic().isEmpty()) { + return Collections.emptyList(); + } + List releasedContainers = request.getReleaseList(); int numReleasedContainers = releasedContainers.size(); if (numReleasedContainers > 0) { @@ -236,8 +232,8 @@ public class OpportunisticContainerAllocator { appContext.getOutstandingOpReqs().descendingKeySet()) { // Allocated containers : // Key = Requested Capability, - // Value = List of Containers of given Cap (The actual container size - // might be different than what is requested.. which is why + // Value = List of Containers of given cap (the actual container size + // might be different than what is requested, which is why // we need the requested capability (key) to match against // the outstanding reqs) Map> allocated = allocate(rmIdentifier, @@ -290,6 +286,10 @@ public class OpportunisticContainerAllocator { } nodesForScheduling.add(nodeEntry.getValue()); } + if (nodesForScheduling.isEmpty()) { + LOG.warn("No nodes available for allocating opportunistic containers."); + return; + } int numAllocated = 0; int nextNodeToSchedule = 0; for (int numCont = 0; numCont < toAllocate; numCont++) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java index 1b701eaaa3a..6fcddf842e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java @@ -18,9 +18,11 @@ package org.apache.hadoop.yarn.server.scheduler; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.NMToken; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -28,9 +30,11 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -56,15 +60,13 @@ public class OpportunisticContainerContext { private ContainerIdGenerator containerIdGenerator = new ContainerIdGenerator(); - private Map nodeMap = new LinkedHashMap<>(); + private volatile List nodeList = new LinkedList<>(); + private final Map nodeMap = new LinkedHashMap<>(); - // Mapping of NodeId to NodeTokens. Populated either from RM response or - // generated locally if required. - private Map nodeTokens = new HashMap<>(); private final Set blacklist = new HashSet<>(); // This maintains a map of outstanding OPPORTUNISTIC Reqs. Key-ed by Priority, - // Resource Name (Host/rack/any) and capability. This mapping is required + // Resource Name (host/rack/any) and capability. This mapping is required // to match a received Container to an outstanding OPPORTUNISTIC // ResourceRequest (ask). private final TreeMap> @@ -74,7 +76,7 @@ public class OpportunisticContainerContext { return containersAllocated; } - public OpportunisticContainerAllocator.AllocationParams getAppParams() { + public AllocationParams getAppParams() { return appParams; } @@ -88,11 +90,29 @@ public class OpportunisticContainerContext { } public Map getNodeMap() { - return nodeMap; + return Collections.unmodifiableMap(nodeMap); } - public Map getNodeTokens() { - return nodeTokens; + public synchronized void updateNodeList(List newNodeList) { + // This is an optimization for centralized placement. The + // OppContainerAllocatorAMService has a cached list of nodes which it sets + // here. The nodeMap needs to be updated only if the backing node list is + // modified. + if (newNodeList != nodeList) { + nodeList = newNodeList; + nodeMap.clear(); + for (NodeId n : nodeList) { + nodeMap.put(n.getHost(), n); + } + } + } + + public void updateAllocationParams(Resource minResource, Resource maxResource, + Resource incrResource, int containerTokenExpiryInterval) { + appParams.setMinResource(minResource); + appParams.setMaxResource(maxResource); + appParams.setIncrementResource(incrResource); + appParams.setContainerTokenExpiryInterval(containerTokenExpiryInterval); } public Set getBlacklist() { @@ -104,6 +124,15 @@ public class OpportunisticContainerContext { return outstandingOpReqs; } + public void updateCompletedContainers(AllocateResponse allocateResponse) { + for (ContainerStatus cs : + allocateResponse.getCompletedContainersStatuses()) { + if (cs.getExecutionType() == ExecutionType.OPPORTUNISTIC) { + containersAllocated.remove(cs.getContainerId()); + } + } + } + /** * Takes a list of ResourceRequests (asks), extracts the key information viz. * (Priority, ResourceName, Capability) and adds to the outstanding diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 7f133340f1c..37f67c458af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -336,8 +336,7 @@ public class NodeManager extends CompositeService addService(nodeHealthChecker); boolean isDistSchedulingEnabled = - conf.getBoolean(YarnConfiguration. - OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, + conf.getBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, YarnConfiguration.DIST_SCHEDULING_ENABLED_DEFAULT); this.context = createNMContext(containerTokenSecretManager, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/DefaultRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/DefaultRequestInterceptor.java index efbdfb4e8ae..22fc8f61014 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/DefaultRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/DefaultRequestInterceptor.java @@ -152,7 +152,7 @@ public final class DefaultRequestInterceptor extends return ((DistributedSchedulingAMProtocol)rmClient) .registerApplicationMasterForDistributedScheduling(request); } else { - throw new YarnException("Distributed Scheduling is not enabled !!"); + throw new YarnException("Distributed Scheduling is not enabled."); } } @@ -174,7 +174,7 @@ public final class DefaultRequestInterceptor extends } return allocateResponse; } else { - throw new YarnException("Distributed Scheduling is not enabled !!"); + throw new YarnException("Distributed Scheduling is not enabled."); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/DistributedScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/DistributedScheduler.java index 368858c43d4..8a40337003e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/DistributedScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/DistributedScheduler.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.nodemanager.scheduler; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateRequest; @@ -32,8 +33,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterReque import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -48,7 +47,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** *

    The DistributedScheduler runs on the NodeManager and is modeled as an @@ -74,6 +75,9 @@ public final class DistributedScheduler extends AbstractRequestInterceptor { private OpportunisticContainerContext oppContainerContext = new OpportunisticContainerContext(); + // Mapping of NodeId to NodeTokens. Populated either from RM response or + // generated locally if required. + private Map nodeTokens = new HashMap<>(); private ApplicationAttemptId applicationAttemptId; private OpportunisticContainerAllocator containerAllocator; private NMTokenSecretManagerInNM nmSecretManager; @@ -157,17 +161,17 @@ public final class DistributedScheduler extends AbstractRequestInterceptor { } /** - * Check if we already have a NMToken. if Not, generate the Token and - * add it to the response + * Adds all the newly allocated Containers to the allocate Response. + * Additionally, in case the NMToken for one of the nodes does not exist, it + * generates one and adds it to the response. */ - private void updateResponseWithNMTokens(AllocateResponse response, + private void updateAllocateResponse(AllocateResponse response, List nmTokens, List allocatedContainers) { List newTokens = new ArrayList<>(); if (allocatedContainers.size() > 0) { response.getAllocatedContainers().addAll(allocatedContainers); for (Container alloc : allocatedContainers) { - if (!oppContainerContext.getNodeTokens().containsKey( - alloc.getNodeId())) { + if (!nodeTokens.containsKey(alloc.getNodeId())) { newTokens.add(nmSecretManager.generateNMToken(appSubmitter, alloc)); } } @@ -179,17 +183,14 @@ public final class DistributedScheduler extends AbstractRequestInterceptor { private void updateParameters( RegisterDistributedSchedulingAMResponse registerResponse) { - oppContainerContext.getAppParams().setMinResource( - registerResponse.getMinContainerResource()); - oppContainerContext.getAppParams().setMaxResource( - registerResponse.getMaxContainerResource()); - oppContainerContext.getAppParams().setIncrementResource( - registerResponse.getIncrContainerResource()); - if (oppContainerContext.getAppParams().getIncrementResource() == null) { - oppContainerContext.getAppParams().setIncrementResource( - oppContainerContext.getAppParams().getMinResource()); + Resource incrementResource = registerResponse.getIncrContainerResource(); + if (incrementResource == null) { + incrementResource = registerResponse.getMinContainerResource(); } - oppContainerContext.getAppParams().setContainerTokenExpiryInterval( + oppContainerContext.updateAllocationParams( + registerResponse.getMinContainerResource(), + registerResponse.getMaxContainerResource(), + incrementResource, registerResponse.getContainerTokenExpiryInterval()); oppContainerContext.getContainerIdGenerator() @@ -198,14 +199,7 @@ public final class DistributedScheduler extends AbstractRequestInterceptor { } private void setNodeList(List nodeList) { - oppContainerContext.getNodeMap().clear(); - addToNodeList(nodeList); - } - - private void addToNodeList(List nodes) { - for (NodeId n : nodes) { - oppContainerContext.getNodeMap().put(n.getHost(), n); - } + oppContainerContext.updateNodeList(nodeList); } @Override @@ -243,23 +237,14 @@ public final class DistributedScheduler extends AbstractRequestInterceptor { setNodeList(dsResp.getNodesForScheduling()); List nmTokens = dsResp.getAllocateResponse().getNMTokens(); for (NMToken nmToken : nmTokens) { - oppContainerContext.getNodeTokens().put(nmToken.getNodeId(), nmToken); + nodeTokens.put(nmToken.getNodeId(), nmToken); } - List completedContainers = - dsResp.getAllocateResponse().getCompletedContainersStatuses(); - - // Only account for opportunistic containers - for (ContainerStatus cs : completedContainers) { - if (cs.getExecutionType() == ExecutionType.OPPORTUNISTIC) { - oppContainerContext.getContainersAllocated() - .remove(cs.getContainerId()); - } - } + oppContainerContext.updateCompletedContainers(dsResp.getAllocateResponse()); // Check if we have NM tokens for all the allocated containers. If not // generate one and update the response. - updateResponseWithNMTokens( + updateAllocateResponse( dsResp.getAllocateResponse(), nmTokens, allocatedContainers); if (LOG.isDebugEnabled()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index ca3847edde5..a9a7e96aa98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -66,6 +66,9 @@ static const int DEFAULT_MIN_USERID = 1000; static const char* DEFAULT_BANNED_USERS[] = {"yarn", "mapred", "hdfs", "bin", 0}; +static const int DEFAULT_DOCKER_SUPPORT_ENABLED = 0; +static const int DEFAULT_TC_SUPPORT_ENABLED = 0; + //location of traffic control binary static const char* TC_BIN = "/sbin/tc"; static const char* TC_MODIFY_STATE_OPTS [] = { "-b" , NULL}; @@ -419,6 +422,43 @@ int change_user(uid_t user, gid_t group) { return 0; } + +static int is_feature_enabled(const char* feature_key, int default_value) { + char *enabled_str = get_value(feature_key, &executor_cfg); + int enabled = default_value; + + if (enabled_str != NULL) { + char *end_ptr = NULL; + enabled = strtol(enabled_str, &end_ptr, 10); + + if ((enabled_str == end_ptr || *end_ptr != '\0') || + (enabled < 0 || enabled > 1)) { + fprintf(LOGFILE, "Illegal value '%s' for '%s' in configuration. " + "Using default value: %d.\n", enabled_str, feature_key, + default_value); + fflush(LOGFILE); + free(enabled_str); + return default_value; + } + + free(enabled_str); + return enabled; + } else { + return default_value; + } +} + + +int is_docker_support_enabled() { + return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY, + DEFAULT_DOCKER_SUPPORT_ENABLED); +} + +int is_tc_support_enabled() { + return is_feature_enabled(TC_SUPPORT_ENABLED_KEY, + DEFAULT_TC_SUPPORT_ENABLED); +} + char* check_docker_binary(char *docker_binary) { if (docker_binary == NULL) { return "docker"; @@ -1136,9 +1176,6 @@ int run_docker(const char *command_file) { snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command); char **args = extract_values_delim(docker_command_with_binary, " "); - //clean up command file before we exec - unlink(command_file); - int exit_code = -1; if (execvp(docker_binary, args) != 0) { fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s", @@ -1451,8 +1488,6 @@ int launch_docker_container_as_user(const char * user, const char *app_id, } cleanup: - //clean up docker command file - unlink(command_file); if (exit_code_file != NULL && write_exit_code_file_as_nm(exit_code_file, exit_code) < 0) { fprintf (ERRORFILE, @@ -2081,7 +2116,6 @@ static int run_traffic_control(const char *opts[], char *command_file) { fprintf(LOGFILE, "failed to execute tc command!\n"); return TRAFFIC_CONTROL_EXECUTION_FAILED; } - unlink(command_file); return 0; } else { execv(TC_BIN, (char**)args); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 18585550ed2..5c17b2962d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -66,7 +66,8 @@ enum errorcodes { TRAFFIC_CONTROL_EXECUTION_FAILED = 28, DOCKER_RUN_FAILED=29, ERROR_OPENING_FILE = 30, - ERROR_READING_FILE = 31 + ERROR_READING_FILE = 31, + FEATURE_DISABLED = 32 }; enum operations { @@ -94,6 +95,8 @@ enum operations { #define BANNED_USERS_KEY "banned.users" #define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users" #define DOCKER_BINARY_KEY "docker.binary" +#define DOCKER_SUPPORT_ENABLED_KEY "feature.docker.enabled" +#define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled" #define TMP_DIR "tmp" extern struct passwd *user_detail; @@ -261,6 +264,9 @@ int check_dir(const char* npath, mode_t st_mode, mode_t desired, int create_validate_dir(const char* npath, mode_t perm, const char* path, int finalComponent); +/** Check if tc (traffic control) support is enabled in configuration. */ +int is_tc_support_enabled(); + /** * Run a batch of tc commands that modify interface configuration */ @@ -280,6 +286,8 @@ int traffic_control_read_state(char *command_file); */ int traffic_control_read_stats(char *command_file); +/** Check if docker support is enabled in configuration. */ +int is_docker_support_enabled(); /** * Run a docker command passing the command file as an argument diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 56215ca4c79..27a269ec954 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -43,27 +43,73 @@ #endif static void display_usage(FILE *stream) { - char *usage_template = - "Usage: container-executor --checksetup\n" \ - " container-executor --mount-cgroups ...\n" \ - " container-executor --tc-modify-state \n" \ - " container-executor --tc-read-state \n" \ - " container-executor --tc-read-stats \n" \ - " container-executor --run-docker \n" \ - " container-executor \n" \ + char usage_template[4096]; + + usage_template[0] = '\0'; + strcat(usage_template, + "Usage: container-executor --checksetup\n" + " container-executor --mount-cgroups " + "...\n" ); + + if(is_tc_support_enabled()) { + strcat(usage_template, + " container-executor --tc-modify-state \n" + " container-executor --tc-read-state \n" + " container-executor --tc-read-stats \n" ); + } else { + strcat(usage_template, + "[DISABLED] container-executor --tc-modify-state \n" + "[DISABLED] container-executor --tc-read-state \n" + "[DISABLED] container-executor --tc-read-stats \n"); + } + + if(is_docker_support_enabled()) { + strcat(usage_template, + " container-executor --run-docker \n"); + } else { + strcat(usage_template, + "[DISABLED] container-executor --run-docker \n"); + } + + strcat(usage_template, + " container-executor \n" " where command and command-args: \n" \ - " initialize container: %2d appid tokens nm-local-dirs nm-log-dirs cmd app...\n" \ - " launch container: %2d appid containerid workdir container-script " \ - "tokens pidfile nm-local-dirs nm-log-dirs resources optional-tc-command-file\n" \ - " launch docker container: %2d appid containerid workdir container-script " \ - "tokens pidfile nm-local-dirs nm-log-dirs docker-command-file resources optional-tc-command-file\n" \ - " signal container: %2d container-pid signal\n" \ - " delete as user: %2d relative-path\n" \ - " list as user: %2d relative-path\n" ; + " initialize container: %2d appid tokens nm-local-dirs " + "nm-log-dirs cmd app...\n" + " launch container: %2d appid containerid workdir " + "container-script tokens pidfile nm-local-dirs nm-log-dirs resources "); + if(is_tc_support_enabled()) { + strcat(usage_template, "optional-tc-command-file\n"); + } else { + strcat(usage_template, "\n"); + } - fprintf(stream, usage_template, INITIALIZE_CONTAINER, LAUNCH_CONTAINER, LAUNCH_DOCKER_CONTAINER, - SIGNAL_CONTAINER, DELETE_AS_USER, LIST_AS_USER); + if(is_docker_support_enabled()) { + strcat(usage_template, + " launch docker container: %2d appid containerid workdir " + "container-script tokens pidfile nm-local-dirs nm-log-dirs " + "docker-command-file resources "); + } else { + strcat(usage_template, + "[DISABLED] launch docker container: %2d appid containerid workdir " + "container-script tokens pidfile nm-local-dirs nm-log-dirs " + "docker-command-file resources "); + } + + if(is_tc_support_enabled()) { + strcat(usage_template, "optional-tc-command-file\n"); + } else { + strcat(usage_template, "\n"); + } + + strcat(usage_template, + " signal container: %2d container-pid signal\n" + " delete as user: %2d relative-path\n" + " list as user: %2d relative-path\n"); + + fprintf(stream, usage_template, INITIALIZE_CONTAINER, LAUNCH_CONTAINER, + LAUNCH_DOCKER_CONTAINER, SIGNAL_CONTAINER, DELETE_AS_USER, LIST_AS_USER); } /* Sets up log files for normal/error logging */ @@ -166,6 +212,10 @@ static void assert_valid_setup(char *argv0) { } +static void display_feature_disabled_message(const char* name) { + fprintf(ERRORFILE, "Feature disabled: %s\n", name); +} + /* Use to store parsed input parmeters for various operations */ static struct { char *cgroups_hierarchy; @@ -222,47 +272,67 @@ static int validate_arguments(int argc, char **argv , int *operation) { } if (strcmp("--tc-modify-state", argv[1]) == 0) { - if (argc != 3) { - display_usage(stdout); - return INVALID_ARGUMENT_NUMBER; + if(is_tc_support_enabled()) { + if (argc != 3) { + display_usage(stdout); + return INVALID_ARGUMENT_NUMBER; + } + optind++; + cmd_input.traffic_control_command_file = argv[optind++]; + *operation = TRAFFIC_CONTROL_MODIFY_STATE; + return 0; + } else { + display_feature_disabled_message("traffic control"); + return FEATURE_DISABLED; } - optind++; - cmd_input.traffic_control_command_file = argv[optind++]; - *operation = TRAFFIC_CONTROL_MODIFY_STATE; - return 0; } if (strcmp("--tc-read-state", argv[1]) == 0) { - if (argc != 3) { - display_usage(stdout); - return INVALID_ARGUMENT_NUMBER; + if(is_tc_support_enabled()) { + if (argc != 3) { + display_usage(stdout); + return INVALID_ARGUMENT_NUMBER; + } + optind++; + cmd_input.traffic_control_command_file = argv[optind++]; + *operation = TRAFFIC_CONTROL_READ_STATE; + return 0; + } else { + display_feature_disabled_message("traffic control"); + return FEATURE_DISABLED; } - optind++; - cmd_input.traffic_control_command_file = argv[optind++]; - *operation = TRAFFIC_CONTROL_READ_STATE; - return 0; } if (strcmp("--tc-read-stats", argv[1]) == 0) { - if (argc != 3) { - display_usage(stdout); - return INVALID_ARGUMENT_NUMBER; + if(is_tc_support_enabled()) { + if (argc != 3) { + display_usage(stdout); + return INVALID_ARGUMENT_NUMBER; + } + optind++; + cmd_input.traffic_control_command_file = argv[optind++]; + *operation = TRAFFIC_CONTROL_READ_STATS; + return 0; + } else { + display_feature_disabled_message("traffic control"); + return FEATURE_DISABLED; } - optind++; - cmd_input.traffic_control_command_file = argv[optind++]; - *operation = TRAFFIC_CONTROL_READ_STATS; - return 0; } if (strcmp("--run-docker", argv[1]) == 0) { - if (argc != 3) { - display_usage(stdout); - return INVALID_ARGUMENT_NUMBER; + if(is_docker_support_enabled()) { + if (argc != 3) { + display_usage(stdout); + return INVALID_ARGUMENT_NUMBER; + } + optind++; + cmd_input.docker_command_file = argv[optind++]; + *operation = RUN_DOCKER; + return 0; + } else { + display_feature_disabled_message("docker"); + return FEATURE_DISABLED; } - optind++; - cmd_input.docker_command_file = argv[optind++]; - *operation = RUN_DOCKER; - return 0; } /* Now we have to validate 'run as user' operations that don't use a 'long option' - we should fix this at some point. The validation/argument @@ -308,51 +378,64 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) *operation = RUN_AS_USER_INITIALIZE_CONTAINER; return 0; case LAUNCH_DOCKER_CONTAINER: - //kill me now. - if (!(argc == 14 || argc == 15)) { - fprintf(ERRORFILE, "Wrong number of arguments (%d vs 14 or 15) for launch docker container\n", - argc); - fflush(ERRORFILE); - return INVALID_ARGUMENT_NUMBER; - } + if(is_docker_support_enabled()) { + //kill me now. + if (!(argc == 14 || argc == 15)) { + fprintf(ERRORFILE, "Wrong number of arguments (%d vs 14 or 15) for" + " launch docker container\n", argc); + fflush(ERRORFILE); + return INVALID_ARGUMENT_NUMBER; + } - cmd_input.app_id = argv[optind++]; - cmd_input.container_id = argv[optind++]; - cmd_input.current_dir = argv[optind++]; - cmd_input.script_file = argv[optind++]; - cmd_input.cred_file = argv[optind++]; - cmd_input.pid_file = argv[optind++]; - cmd_input.local_dirs = argv[optind++];// good local dirs as a comma separated list - cmd_input.log_dirs = argv[optind++];// good log dirs as a comma separated list - cmd_input.docker_command_file = argv[optind++]; - resources = argv[optind++];// key,value pair describing resources - resources_key = malloc(strlen(resources)); - resources_value = malloc(strlen(resources)); - if (get_kv_key(resources, resources_key, strlen(resources)) < 0 || - get_kv_value(resources, resources_value, strlen(resources)) < 0) { - fprintf(ERRORFILE, "Invalid arguments for cgroups resources: %s", - resources); - fflush(ERRORFILE); - free(resources_key); - free(resources_value); - return INVALID_ARGUMENT_NUMBER; - } - //network isolation through tc - if (argc == 15) { - cmd_input.traffic_control_command_file = argv[optind++]; - } + cmd_input.app_id = argv[optind++]; + cmd_input.container_id = argv[optind++]; + cmd_input.current_dir = argv[optind++]; + cmd_input.script_file = argv[optind++]; + cmd_input.cred_file = argv[optind++]; + cmd_input.pid_file = argv[optind++]; + // good local dirs as a comma separated list + cmd_input.local_dirs = argv[optind++]; + // good log dirs as a comma separated list + cmd_input.log_dirs = argv[optind++]; + cmd_input.docker_command_file = argv[optind++]; + // key,value pair describing resources + resources = argv[optind++]; + resources_key = malloc(strlen(resources)); + resources_value = malloc(strlen(resources)); + if (get_kv_key(resources, resources_key, strlen(resources)) < 0 || + get_kv_value(resources, resources_value, strlen(resources)) < 0) { + fprintf(ERRORFILE, "Invalid arguments for cgroups resources: %s", + resources); + fflush(ERRORFILE); + free(resources_key); + free(resources_value); + return INVALID_ARGUMENT_NUMBER; + } + //network isolation through tc + if (argc == 15) { + if(is_tc_support_enabled()) { + cmd_input.traffic_control_command_file = argv[optind++]; + } else { + display_feature_disabled_message("traffic control"); + return FEATURE_DISABLED; + } + } - cmd_input.resources_key = resources_key; - cmd_input.resources_value = resources_value; - cmd_input.resources_values = extract_values(resources_value); - *operation = RUN_AS_USER_LAUNCH_DOCKER_CONTAINER; - return 0; + cmd_input.resources_key = resources_key; + cmd_input.resources_value = resources_value; + cmd_input.resources_values = extract_values(resources_value); + *operation = RUN_AS_USER_LAUNCH_DOCKER_CONTAINER; + return 0; + } else { + display_feature_disabled_message("docker"); + return FEATURE_DISABLED; + } case LAUNCH_CONTAINER: //kill me now. if (!(argc == 13 || argc == 14)) { - fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13 or 14) for launch container\n", - argc); + fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13 or 14)" + " for launch container\n", argc); fflush(ERRORFILE); return INVALID_ARGUMENT_NUMBER; } @@ -381,7 +464,12 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) //network isolation through tc if (argc == 14) { - cmd_input.traffic_control_command_file = argv[optind++]; + if(is_tc_support_enabled()) { + cmd_input.traffic_control_command_file = argv[optind++]; + } else { + display_feature_disabled_message("traffic control"); + return FEATURE_DISABLED; + } } cmd_input.resources_key = resources_key; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index db55264c483..33daf7f09bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -21,6 +21,9 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Set; @@ -806,6 +809,49 @@ public class AdminService extends CompositeService implements ReplaceLabelsOnNodeResponse response = recordFactory.newRecordInstance(ReplaceLabelsOnNodeResponse.class); + + if (request.getFailOnUnknownNodes()) { + // verify if nodes have registered to RM + List unknownNodes = new ArrayList<>(); + for (NodeId requestedNode : request.getNodeToLabels().keySet()) { + boolean isKnown = false; + // both active and inactive nodes are recognized as known nodes + if (requestedNode.getPort() != 0) { + if (rmContext.getRMNodes().containsKey(requestedNode) + || rmContext.getInactiveRMNodes().containsKey(requestedNode)) { + isKnown = true; + } + } else { + for (NodeId knownNode : rmContext.getRMNodes().keySet()) { + if (knownNode.getHost().equals(requestedNode.getHost())) { + isKnown = true; + break; + } + } + if (!isKnown) { + for (NodeId knownNode : rmContext.getInactiveRMNodes().keySet()) { + if (knownNode.getHost().equals(requestedNode.getHost())) { + isKnown = true; + break; + } + } + } + } + if (!isKnown) { + unknownNodes.add(requestedNode); + } + } + + if (!unknownNodes.isEmpty()) { + RMAuditLogger.logFailure(user.getShortUserName(), operation, "", + "AdminService", + "Failed to replace labels as there are unknown nodes:" + + Arrays.toString(unknownNodes.toArray())); + throw RPCUtil.getRemoteException(new IOException( + "Failed to replace labels as there are unknown nodes:" + + Arrays.toString(unknownNodes.toArray()))); + } + } try { rmContext.getNodeLabelManager().replaceLabelsOnNode( request.getNodeToLabels()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java index 72327e82e9c..88d2e102562 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager; +import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.InvalidProtocolBufferException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -39,6 +40,8 @@ import org.apache.zookeeper.data.ACL; import java.io.IOException; import java.util.List; +import java.util.Timer; +import java.util.TimerTask; @InterfaceAudience.Private @InterfaceStability.Unstable @@ -54,6 +57,10 @@ public class EmbeddedElectorService extends AbstractService private byte[] localActiveNodeInfo; private ActiveStandbyElector elector; + private long zkSessionTimeout; + private Timer zkDisconnectTimer; + @VisibleForTesting + final Object zkDisconnectLock = new Object(); EmbeddedElectorService(RMContext rmContext) { super(EmbeddedElectorService.class.getName()); @@ -80,7 +87,7 @@ public class EmbeddedElectorService extends AbstractService YarnConfiguration.DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH); String electionZNode = zkBasePath + "/" + clusterId; - long zkSessionTimeout = conf.getLong(YarnConfiguration.RM_ZK_TIMEOUT_MS, + zkSessionTimeout = conf.getLong(YarnConfiguration.RM_ZK_TIMEOUT_MS, YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS); List zkAcls = RMZKUtils.getZKAcls(conf); @@ -123,6 +130,8 @@ public class EmbeddedElectorService extends AbstractService @Override public void becomeActive() throws ServiceFailedException { + cancelDisconnectTimer(); + try { rmContext.getRMAdminService().transitionToActive(req); } catch (Exception e) { @@ -132,6 +141,8 @@ public class EmbeddedElectorService extends AbstractService @Override public void becomeStandby() { + cancelDisconnectTimer(); + try { rmContext.getRMAdminService().transitionToStandby(req); } catch (Exception e) { @@ -139,13 +150,49 @@ public class EmbeddedElectorService extends AbstractService } } + /** + * Stop the disconnect timer. Any running tasks will be allowed to complete. + */ + private void cancelDisconnectTimer() { + synchronized (zkDisconnectLock) { + if (zkDisconnectTimer != null) { + zkDisconnectTimer.cancel(); + zkDisconnectTimer = null; + } + } + } + + /** + * When the ZK client loses contact with ZK, this method will be called to + * allow the RM to react. Because the loss of connection can be noticed + * before the session timeout happens, it is undesirable to transition + * immediately. Instead the method starts a timer that will wait + * {@link YarnConfiguration#RM_ZK_TIMEOUT_MS} milliseconds before + * initiating the transition into standby state. + */ @Override public void enterNeutralMode() { - /** - * Possibly due to transient connection issues. Do nothing. - * TODO: Might want to keep track of how long in this state and transition - * to standby. - */ + LOG.warn("Lost contact with Zookeeper. Transitioning to standby in " + + zkSessionTimeout + " ms if connection is not reestablished."); + + // If we've just become disconnected, start a timer. When the time's up, + // we'll transition to standby. + synchronized (zkDisconnectLock) { + if (zkDisconnectTimer == null) { + zkDisconnectTimer = new Timer("Zookeeper disconnect timer"); + zkDisconnectTimer.schedule(new TimerTask() { + @Override + public void run() { + synchronized (zkDisconnectLock) { + // Only run if the timer hasn't been cancelled + if (zkDisconnectTimer != null) { + becomeStandby(); + } + } + } + }, zkSessionTimeout); + } + } } @SuppressWarnings(value = "unchecked") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java index a473b14d0da..a7c0a507ca1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java @@ -24,9 +24,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocol; import org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl; @@ -65,12 +67,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateS import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; + +import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator; +import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext; +import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils; + import java.io.IOException; import java.net.InetSocketAddress; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; /** * The OpportunisticContainerAllocatorAMService is started instead of the @@ -88,17 +92,20 @@ public class OpportunisticContainerAllocatorAMService LogFactory.getLog(OpportunisticContainerAllocatorAMService.class); private final NodeQueueLoadMonitor nodeMonitor; + private final OpportunisticContainerAllocator oppContainerAllocator; - private final ConcurrentHashMap> rackToNode = - new ConcurrentHashMap<>(); - private final ConcurrentHashMap> hostToNode = - new ConcurrentHashMap<>(); private final int k; + private final long cacheRefreshInterval; + private List cachedNodeIds; + private long lastCacheUpdateTime; + public OpportunisticContainerAllocatorAMService(RMContext rmContext, YarnScheduler scheduler) { super(OpportunisticContainerAllocatorAMService.class.getName(), rmContext, scheduler); + this.oppContainerAllocator = new OpportunisticContainerAllocator( + rmContext.getContainerTokenSecretManager(), 0); this.k = rmContext.getYarnConfiguration().getInt( YarnConfiguration.OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED, YarnConfiguration.OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED_DEFAULT); @@ -106,6 +113,8 @@ public class OpportunisticContainerAllocatorAMService YarnConfiguration.NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS, YarnConfiguration. NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS_DEFAULT); + this.cacheRefreshInterval = nodeSortInterval; + this.lastCacheUpdateTime = System.currentTimeMillis(); NodeQueueLoadMonitor.LoadComparator comparator = NodeQueueLoadMonitor.LoadComparator.valueOf( rmContext.getYarnConfiguration().get( @@ -172,6 +181,27 @@ public class OpportunisticContainerAllocatorAMService public RegisterApplicationMasterResponse registerApplicationMaster (RegisterApplicationMasterRequest request) throws YarnException, IOException { + final ApplicationAttemptId appAttemptId = getAppAttemptId(); + SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler) + rmContext.getScheduler()).getApplicationAttempt(appAttemptId); + if (appAttempt.getOpportunisticContainerContext() == null) { + OpportunisticContainerContext opCtx = new OpportunisticContainerContext(); + opCtx.setContainerIdGenerator(new OpportunisticContainerAllocator + .ContainerIdGenerator() { + @Override + public long generateContainerId() { + return appAttempt.getAppSchedulingInfo().getNewContainerId(); + } + }); + int tokenExpiryInterval = getConfig() + .getInt(YarnConfiguration.OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS, + YarnConfiguration. + OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT); + opCtx.updateAllocationParams(createMinContainerResource(), + createMaxContainerResource(), createIncrContainerResource(), + tokenExpiryInterval); + appAttempt.setOpportunisticContainerContext(opCtx); + } return super.registerApplicationMaster(request); } @@ -185,7 +215,30 @@ public class OpportunisticContainerAllocatorAMService @Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { - return super.allocate(request); + + final ApplicationAttemptId appAttemptId = getAppAttemptId(); + SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler) + rmContext.getScheduler()).getApplicationAttempt(appAttemptId); + OpportunisticContainerContext oppCtx = + appAttempt.getOpportunisticContainerContext(); + oppCtx.updateNodeList(getLeastLoadedNodes()); + List oppContainers = + oppContainerAllocator.allocateContainers(request, appAttemptId, oppCtx, + ResourceManager.getClusterTimeStamp(), appAttempt.getUser()); + + if (!oppContainers.isEmpty()) { + handleNewContainers(oppContainers, false); + appAttempt.updateNMTokens(oppContainers); + } + + // Allocate all guaranteed containers + AllocateResponse allocateResp = super.allocate(request); + + oppCtx.updateCompletedContainers(allocateResp); + + // Add all opportunistic containers + allocateResp.getAllocatedContainers().addAll(oppContainers); + return allocateResp; } @Override @@ -198,39 +251,9 @@ public class OpportunisticContainerAllocatorAMService RegisterDistributedSchedulingAMResponse dsResp = recordFactory .newRecordInstance(RegisterDistributedSchedulingAMResponse.class); dsResp.setRegisterResponse(response); - dsResp.setMinContainerResource( - Resource.newInstance( - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB, - YarnConfiguration. - OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB_DEFAULT), - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES, - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES_DEFAULT) - ) - ); - dsResp.setMaxContainerResource( - Resource.newInstance( - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB, - YarnConfiguration - .OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB_DEFAULT), - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES, - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES_DEFAULT) - ) - ); - dsResp.setIncrContainerResource( - Resource.newInstance( - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB, - YarnConfiguration. - OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB_DEFAULT), - getConfig().getInt( - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES, - YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES_DEFAULT) - ) - ); + dsResp.setMinContainerResource(createMinContainerResource()); + dsResp.setMaxContainerResource(createMaxContainerResource()); + dsResp.setIncrContainerResource(createIncrContainerResource()); dsResp.setContainerTokenExpiryInterval( getConfig().getInt( YarnConfiguration.OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS, @@ -240,8 +263,7 @@ public class OpportunisticContainerAllocatorAMService this.rmContext.getEpoch() << ResourceManager.EPOCH_BIT_SHIFT); // Set nodes to be used for scheduling - dsResp.setNodesForScheduling( - this.nodeMonitor.selectLeastLoadedNodes(this.k)); + dsResp.setNodesForScheduling(getLeastLoadedNodes()); return dsResp; } @@ -250,47 +272,30 @@ public class OpportunisticContainerAllocatorAMService DistributedSchedulingAllocateRequest request) throws YarnException, IOException { List distAllocContainers = request.getAllocatedContainers(); - for (Container container : distAllocContainers) { + handleNewContainers(distAllocContainers, true); + AllocateResponse response = allocate(request.getAllocateRequest()); + DistributedSchedulingAllocateResponse dsResp = recordFactory + .newRecordInstance(DistributedSchedulingAllocateResponse.class); + dsResp.setAllocateResponse(response); + dsResp.setNodesForScheduling(getLeastLoadedNodes()); + return dsResp; + } + + private void handleNewContainers(List allocContainers, + boolean isRemotelyAllocated) { + for (Container container : allocContainers) { // Create RMContainer SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler) rmContext.getScheduler()) .getCurrentAttemptForContainer(container.getId()); RMContainer rmContainer = new RMContainerImpl(container, appAttempt.getApplicationAttemptId(), container.getNodeId(), - appAttempt.getUser(), rmContext, true); + appAttempt.getUser(), rmContext, isRemotelyAllocated); appAttempt.addRMContainer(container.getId(), rmContainer); rmContainer.handle( new RMContainerEvent(container.getId(), RMContainerEventType.LAUNCHED)); } - AllocateResponse response = allocate(request.getAllocateRequest()); - DistributedSchedulingAllocateResponse dsResp = recordFactory - .newRecordInstance(DistributedSchedulingAllocateResponse.class); - dsResp.setAllocateResponse(response); - dsResp.setNodesForScheduling( - this.nodeMonitor.selectLeastLoadedNodes(this.k)); - return dsResp; - } - - private void addToMapping(ConcurrentHashMap> mapping, - String rackName, NodeId nodeId) { - if (rackName != null) { - mapping.putIfAbsent(rackName, new HashSet()); - Set nodeIds = mapping.get(rackName); - synchronized (nodeIds) { - nodeIds.add(nodeId); - } - } - } - - private void removeFromMapping(ConcurrentHashMap> mapping, - String rackName, NodeId nodeId) { - if (rackName != null) { - Set nodeIds = mapping.get(rackName); - synchronized (nodeIds) { - nodeIds.remove(nodeId); - } - } } @Override @@ -303,10 +308,6 @@ public class OpportunisticContainerAllocatorAMService NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent) event; nodeMonitor.addNode(nodeAddedEvent.getContainerReports(), nodeAddedEvent.getAddedRMNode()); - addToMapping(rackToNode, nodeAddedEvent.getAddedRMNode().getRackName(), - nodeAddedEvent.getAddedRMNode().getNodeID()); - addToMapping(hostToNode, nodeAddedEvent.getAddedRMNode().getHostName(), - nodeAddedEvent.getAddedRMNode().getNodeID()); break; case NODE_REMOVED: if (!(event instanceof NodeRemovedSchedulerEvent)) { @@ -315,12 +316,6 @@ public class OpportunisticContainerAllocatorAMService NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent) event; nodeMonitor.removeNode(nodeRemovedEvent.getRemovedRMNode()); - removeFromMapping(rackToNode, - nodeRemovedEvent.getRemovedRMNode().getRackName(), - nodeRemovedEvent.getRemovedRMNode().getNodeID()); - removeFromMapping(hostToNode, - nodeRemovedEvent.getRemovedRMNode().getHostName(), - nodeRemovedEvent.getRemovedRMNode().getNodeID()); break; case NODE_UPDATE: if (!(event instanceof NodeUpdateSchedulerEvent)) { @@ -364,4 +359,58 @@ public class OpportunisticContainerAllocatorAMService public QueueLimitCalculator getNodeManagerQueueLimitCalculator() { return nodeMonitor.getThresholdCalculator(); } + + private Resource createIncrContainerResource() { + return Resource.newInstance( + getConfig().getInt( + YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB, + YarnConfiguration. + OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB_DEFAULT), + getConfig().getInt( + YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES, + YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES_DEFAULT) + ); + } + + private synchronized List getLeastLoadedNodes() { + long currTime = System.currentTimeMillis(); + if ((currTime - lastCacheUpdateTime > cacheRefreshInterval) + || cachedNodeIds == null) { + cachedNodeIds = this.nodeMonitor.selectLeastLoadedNodes(this.k); + lastCacheUpdateTime = currTime; + } + return cachedNodeIds; + } + + private Resource createMaxContainerResource() { + return Resource.newInstance( + getConfig().getInt( + YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB, + YarnConfiguration + .OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB_DEFAULT), + getConfig().getInt( + YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES, + YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES_DEFAULT) + ); + } + + private Resource createMinContainerResource() { + return Resource.newInstance( + getConfig().getInt( + YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB, + YarnConfiguration. + OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB_DEFAULT), + getConfig().getInt( + YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES, + YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES_DEFAULT) + ); + } + + private static ApplicationAttemptId getAppAttemptId() throws YarnException { + AMRMTokenIdentifier amrmTokenIdentifier = + YarnServerSecurityUtils.authorizeRequest(); + ApplicationAttemptId applicationAttemptId = + amrmTokenIdentifier.getApplicationAttemptId(); + return applicationAttemptId; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 7352a284c48..c065b607795 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -101,7 +101,7 @@ public class RMAppManager implements EventHandler, this.maxCompletedAppsInStateStore = conf.getInt( YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS, - YarnConfiguration.DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS); + this.maxCompletedAppsInMemory); if (this.maxCompletedAppsInStateStore > this.maxCompletedAppsInMemory) { this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index b90e499601d..b2a085a479f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -211,10 +211,7 @@ public class RMServerUtils { } /** - * Validate increase/decrease request. This function must be called under - * the queue lock to make sure that the access to container resource is - * atomic. Refer to LeafQueue.decreaseContainer() and - * CapacityScheduelr.updateIncreaseRequests() + * Validate increase/decrease request. * 

        * - Throw exception when any other error happens
    *
    diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 5e9bece33c3..d2d706d05a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -1184,6 +1184,13 @@ public class ResourceManager extends CompositeService implements Recoverable { Configuration config = this.rmContext.getYarnConfiguration(); if (YarnConfiguration.isOpportunisticContainerAllocationEnabled(config) || YarnConfiguration.isDistSchedulingEnabled(config)) { + if (YarnConfiguration.isDistSchedulingEnabled(config) && + !YarnConfiguration + .isOpportunisticContainerAllocationEnabled(config)) { + throw new YarnRuntimeException( + "Invalid parameters: opportunistic container allocation has to " + + "be enabled when distributed scheduling is enabled."); + } OpportunisticContainerAllocatorAMService oppContainerAllocatingAMService = new OpportunisticContainerAllocatorAMService(this.rmContext, @@ -1193,9 +1200,8 @@ public class ResourceManager extends CompositeService implements Recoverable { OpportunisticContainerAllocatorAMService.class.getName()); // Add an event dispatcher for the // OpportunisticContainerAllocatorAMService to handle node - // updates/additions and removals. - // Since the SchedulerEvent is currently a super set of theses, - // we register interest for it.. + // additions, updates and removals. Since the SchedulerEvent is currently + // a super set of theses, we register interest for it. addService(oppContainerAllocEventDispatcher); rmDispatcher.register(SchedulerEventType.class, oppContainerAllocEventDispatcher); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 727703b690e..0fdc3113fa3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -1061,7 +1061,7 @@ public class RMAppImpl implements RMApp, Recoverable { } app.rmContext.getSystemMetricsPublisher().appUpdated(app, - System.currentTimeMillis()); + app.systemClock.getTime()); // TODO: Write out change to state store (YARN-1558) // Also take care of RM failover diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 45415de7e98..645e06dbf54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -72,8 +73,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerReco import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; @@ -94,7 +93,7 @@ public abstract class AbstractYarnScheduler protected Resource minimumAllocation; - protected RMContext rmContext; + protected volatile RMContext rmContext; private volatile Priority maxClusterLevelAppPriority; @@ -112,6 +111,18 @@ public abstract class AbstractYarnScheduler protected static final Allocation EMPTY_ALLOCATION = new Allocation( EMPTY_CONTAINER_LIST, Resources.createResource(0), null, null, null); + protected final ReentrantReadWriteLock.ReadLock readLock; + + /* + * Use writeLock for any of operations below: + * - queue change (hierarchy / configuration / container allocation) + * - application(add/remove/allocate-container, but not include container + * finish) + * - node (add/remove/change-resource/container-allocation, but not include + * container finish) + */ + protected final ReentrantReadWriteLock.WriteLock writeLock; + /** * Construct the service. * @@ -119,6 +130,9 @@ public abstract class AbstractYarnScheduler */ public AbstractYarnScheduler(String name) { super(name); + ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + readLock = lock.readLock(); + writeLock = lock.writeLock(); } @Override @@ -141,6 +155,10 @@ public abstract class AbstractYarnScheduler return nodeTracker; } + /* + * YARN-3136 removed synchronized lock for this method for performance + * purposes + */ public List getTransferredContainers( ApplicationAttemptId currentAttempt) { ApplicationId appId = currentAttempt.getApplicationId(); @@ -155,9 +173,8 @@ public abstract class AbstractYarnScheduler } Collection liveContainers = app.getCurrentAppAttempt().getLiveContainers(); - ContainerId amContainerId = - rmContext.getRMApps().get(appId).getCurrentAppAttempt() - .getMasterContainer().getId(); + ContainerId amContainerId = rmContext.getRMApps().get(appId) + .getCurrentAppAttempt().getMasterContainer().getId(); for (RMContainer rmContainer : liveContainers) { if (!rmContainer.getContainerId().equals(amContainerId)) { containerList.add(rmContainer.getContainer()); @@ -211,54 +228,59 @@ public abstract class AbstractYarnScheduler nodeTracker.setConfiguredMaxAllocation(maximumAllocation); } - protected synchronized void containerLaunchedOnNode( + protected void containerLaunchedOnNode( ContainerId containerId, SchedulerNode node) { - // Get the application for the finished container - SchedulerApplicationAttempt application = - getCurrentAttemptForContainer(containerId); - if (application == null) { - LOG.info("Unknown application " + containerId.getApplicationAttemptId() - .getApplicationId() + " launched container " + containerId - + " on node: " + node); - this.rmContext.getDispatcher().getEventHandler() - .handle(new RMNodeCleanContainerEvent(node.getNodeID(), containerId)); - return; - } + try { + readLock.lock(); + // Get the application for the finished container + SchedulerApplicationAttempt application = getCurrentAttemptForContainer( + containerId); + if (application == null) { + LOG.info("Unknown application " + containerId.getApplicationAttemptId() + .getApplicationId() + " launched container " + containerId + + " on node: " + node); + this.rmContext.getDispatcher().getEventHandler().handle( + new RMNodeCleanContainerEvent(node.getNodeID(), containerId)); + return; + } - application.containerLaunchedOnNode(containerId, node.getNodeID()); + application.containerLaunchedOnNode(containerId, node.getNodeID()); + } finally { + readLock.unlock(); + } } protected void containerIncreasedOnNode(ContainerId containerId, SchedulerNode node, Container increasedContainerReportedByNM) { + /* + * No lock is required, as this method is protected by scheduler's writeLock + */ // Get the application for the finished container - SchedulerApplicationAttempt application = - getCurrentAttemptForContainer(containerId); + SchedulerApplicationAttempt application = getCurrentAttemptForContainer( + containerId); if (application == null) { - LOG.info("Unknown application " - + containerId.getApplicationAttemptId().getApplicationId() - + " increased container " + containerId + " on node: " + node); - this.rmContext.getDispatcher().getEventHandler() - .handle(new RMNodeCleanContainerEvent(node.getNodeID(), containerId)); + LOG.info("Unknown application " + containerId.getApplicationAttemptId() + .getApplicationId() + " increased container " + containerId + + " on node: " + node); + this.rmContext.getDispatcher().getEventHandler().handle( + new RMNodeCleanContainerEvent(node.getNodeID(), containerId)); return; } - LeafQueue leafQueue = (LeafQueue) application.getQueue(); - synchronized (leafQueue) { - RMContainer rmContainer = getRMContainer(containerId); - if (rmContainer == null) { - // Some unknown container sneaked into the system. Kill it. - this.rmContext.getDispatcher().getEventHandler() - .handle(new RMNodeCleanContainerEvent( - node.getNodeID(), containerId)); - return; - } - rmContainer.handle(new RMContainerNMDoneChangeResourceEvent( - containerId, increasedContainerReportedByNM.getResource())); + + RMContainer rmContainer = getRMContainer(containerId); + if (rmContainer == null) { + // Some unknown container sneaked into the system. Kill it. + this.rmContext.getDispatcher().getEventHandler().handle( + new RMNodeCleanContainerEvent(node.getNodeID(), containerId)); + return; } + rmContainer.handle(new RMContainerNMDoneChangeResourceEvent(containerId, + increasedContainerReportedByNM.getResource())); } public T getApplicationAttempt(ApplicationAttemptId applicationAttemptId) { - SchedulerApplication app = - applications.get(applicationAttemptId.getApplicationId()); + SchedulerApplication app = applications.get( + applicationAttemptId.getApplicationId()); return app == null ? null : app.getCurrentAppAttempt(); } @@ -338,96 +360,101 @@ public abstract class AbstractYarnScheduler } } - public synchronized void recoverContainersOnNode( + public void recoverContainersOnNode( List containerReports, RMNode nm) { - if (!rmContext.isWorkPreservingRecoveryEnabled() - || containerReports == null - || (containerReports != null && containerReports.isEmpty())) { - return; - } - - for (NMContainerStatus container : containerReports) { - ApplicationId appId = - container.getContainerId().getApplicationAttemptId().getApplicationId(); - RMApp rmApp = rmContext.getRMApps().get(appId); - if (rmApp == null) { - LOG.error("Skip recovering container " + container - + " for unknown application."); - killOrphanContainerOnNode(nm, container); - continue; + try { + writeLock.lock(); + if (!rmContext.isWorkPreservingRecoveryEnabled() + || containerReports == null || (containerReports != null + && containerReports.isEmpty())) { + return; } - SchedulerApplication schedulerApp = applications.get(appId); - if (schedulerApp == null) { - LOG.info("Skip recovering container " + container - + " for unknown SchedulerApplication. Application current state is " - + rmApp.getState()); - killOrphanContainerOnNode(nm, container); - continue; - } - - LOG.info("Recovering container " + container); - SchedulerApplicationAttempt schedulerAttempt = - schedulerApp.getCurrentAppAttempt(); - - if (!rmApp.getApplicationSubmissionContext() - .getKeepContainersAcrossApplicationAttempts()) { - // Do not recover containers for stopped attempt or previous attempt. - if (schedulerAttempt.isStopped() - || !schedulerAttempt.getApplicationAttemptId().equals( - container.getContainerId().getApplicationAttemptId())) { - LOG.info("Skip recovering container " + container - + " for already stopped attempt."); + for (NMContainerStatus container : containerReports) { + ApplicationId appId = + container.getContainerId().getApplicationAttemptId() + .getApplicationId(); + RMApp rmApp = rmContext.getRMApps().get(appId); + if (rmApp == null) { + LOG.error("Skip recovering container " + container + + " for unknown application."); killOrphanContainerOnNode(nm, container); continue; } - } - // create container - RMContainer rmContainer = recoverAndCreateContainer(container, nm); - - // recover RMContainer - rmContainer.handle(new RMContainerRecoverEvent(container.getContainerId(), - container)); - - // recover scheduler node - SchedulerNode schedulerNode = nodeTracker.getNode(nm.getNodeID()); - schedulerNode.recoverContainer(rmContainer); - - // recover queue: update headroom etc. - Queue queue = schedulerAttempt.getQueue(); - queue.recoverContainer( - getClusterResource(), schedulerAttempt, rmContainer); - - // recover scheduler attempt - schedulerAttempt.recoverContainer(schedulerNode, rmContainer); - - // set master container for the current running AMContainer for this - // attempt. - RMAppAttempt appAttempt = rmApp.getCurrentAppAttempt(); - if (appAttempt != null) { - Container masterContainer = appAttempt.getMasterContainer(); - - // Mark current running AMContainer's RMContainer based on the master - // container ID stored in AppAttempt. - if (masterContainer != null - && masterContainer.getId().equals(rmContainer.getContainerId())) { - ((RMContainerImpl)rmContainer).setAMContainer(true); + SchedulerApplication schedulerApp = applications.get(appId); + if (schedulerApp == null) { + LOG.info("Skip recovering container " + container + + " for unknown SchedulerApplication. " + + "Application current state is " + rmApp.getState()); + killOrphanContainerOnNode(nm, container); + continue; } - } - synchronized (schedulerAttempt) { - Set releases = schedulerAttempt.getPendingRelease(); - if (releases.contains(container.getContainerId())) { + LOG.info("Recovering container " + container); + SchedulerApplicationAttempt schedulerAttempt = + schedulerApp.getCurrentAppAttempt(); + + if (!rmApp.getApplicationSubmissionContext() + .getKeepContainersAcrossApplicationAttempts()) { + // Do not recover containers for stopped attempt or previous attempt. + if (schedulerAttempt.isStopped() || !schedulerAttempt + .getApplicationAttemptId().equals( + container.getContainerId().getApplicationAttemptId())) { + LOG.info("Skip recovering container " + container + + " for already stopped attempt."); + killOrphanContainerOnNode(nm, container); + continue; + } + } + + // create container + RMContainer rmContainer = recoverAndCreateContainer(container, nm); + + // recover RMContainer + rmContainer.handle( + new RMContainerRecoverEvent(container.getContainerId(), container)); + + // recover scheduler node + SchedulerNode schedulerNode = nodeTracker.getNode(nm.getNodeID()); + schedulerNode.recoverContainer(rmContainer); + + // recover queue: update headroom etc. + Queue queue = schedulerAttempt.getQueue(); + queue.recoverContainer(getClusterResource(), schedulerAttempt, + rmContainer); + + // recover scheduler attempt + schedulerAttempt.recoverContainer(schedulerNode, rmContainer); + + // set master container for the current running AMContainer for this + // attempt. + RMAppAttempt appAttempt = rmApp.getCurrentAppAttempt(); + if (appAttempt != null) { + Container masterContainer = appAttempt.getMasterContainer(); + + // Mark current running AMContainer's RMContainer based on the master + // container ID stored in AppAttempt. + if (masterContainer != null && masterContainer.getId().equals( + rmContainer.getContainerId())) { + ((RMContainerImpl) rmContainer).setAMContainer(true); + } + } + + if (schedulerAttempt.getPendingRelease().remove( + container.getContainerId())) { // release the container - rmContainer.handle(new RMContainerFinishedEvent(container - .getContainerId(), SchedulerUtils.createAbnormalContainerStatus( - container.getContainerId(), SchedulerUtils.RELEASED_CONTAINER), - RMContainerEventType.RELEASED)); - releases.remove(container.getContainerId()); + rmContainer.handle( + new RMContainerFinishedEvent(container.getContainerId(), + SchedulerUtils + .createAbnormalContainerStatus(container.getContainerId(), + SchedulerUtils.RELEASED_CONTAINER), + RMContainerEventType.RELEASED)); LOG.info(container.getContainerId() + " is released by application."); } } + } finally { + writeLock.unlock(); } } @@ -492,17 +519,15 @@ public abstract class AbstractYarnScheduler for (SchedulerApplication app : applications.values()) { T attempt = app.getCurrentAppAttempt(); if (attempt != null) { - synchronized (attempt) { - for (ContainerId containerId : attempt.getPendingRelease()) { - RMAuditLogger.logFailure(app.getUser(), - AuditConstants.RELEASE_CONTAINER, - "Unauthorized access or invalid container", "Scheduler", - "Trying to release container not owned by app " - + "or with invalid id.", attempt.getApplicationId(), - containerId, null); - } - attempt.getPendingRelease().clear(); + for (ContainerId containerId : attempt.getPendingRelease()) { + RMAuditLogger.logFailure(app.getUser(), + AuditConstants.RELEASE_CONTAINER, + "Unauthorized access or invalid container", "Scheduler", + "Trying to release container not owned by app " + + "or with invalid id.", attempt.getApplicationId(), + containerId, null); } + attempt.getPendingRelease().clear(); } } } @@ -558,9 +583,7 @@ public abstract class AbstractYarnScheduler < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); - synchronized (attempt) { - attempt.getPendingRelease().add(containerId); - } + attempt.getPendingRelease().add(containerId); } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, @@ -603,81 +626,92 @@ public abstract class AbstractYarnScheduler } @Override - public synchronized void moveAllApps(String sourceQueue, String destQueue) + public void moveAllApps(String sourceQueue, String destQueue) throws YarnException { - // check if destination queue is a valid leaf queue try { - getQueueInfo(destQueue, false, false); - } catch (IOException e) { - LOG.warn(e); - throw new YarnException(e); - } - // check if source queue is a valid - List apps = getAppsInQueue(sourceQueue); - if (apps == null) { - String errMsg = "The specified Queue: " + sourceQueue + " doesn't exist"; - LOG.warn(errMsg); - throw new YarnException(errMsg); - } - // generate move events for each pending/running app - for (ApplicationAttemptId app : apps) { - SettableFuture future = SettableFuture.create(); - this.rmContext - .getDispatcher() - .getEventHandler() - .handle(new RMAppMoveEvent(app.getApplicationId(), destQueue, future)); + writeLock.lock(); + // check if destination queue is a valid leaf queue + try { + getQueueInfo(destQueue, false, false); + } catch (IOException e) { + LOG.warn(e); + throw new YarnException(e); + } + // check if source queue is a valid + List apps = getAppsInQueue(sourceQueue); + if (apps == null) { + String errMsg = + "The specified Queue: " + sourceQueue + " doesn't exist"; + LOG.warn(errMsg); + throw new YarnException(errMsg); + } + // generate move events for each pending/running app + for (ApplicationAttemptId app : apps) { + SettableFuture future = SettableFuture.create(); + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppMoveEvent(app.getApplicationId(), destQueue, future)); + } + } finally { + writeLock.unlock(); } } @Override - public synchronized void killAllAppsInQueue(String queueName) + public void killAllAppsInQueue(String queueName) throws YarnException { - // check if queue is a valid - List apps = getAppsInQueue(queueName); - if (apps == null) { - String errMsg = "The specified Queue: " + queueName + " doesn't exist"; - LOG.warn(errMsg); - throw new YarnException(errMsg); - } - // generate kill events for each pending/running app - for (ApplicationAttemptId app : apps) { - this.rmContext - .getDispatcher() - .getEventHandler() - .handle(new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL, - "Application killed due to expiry of reservation queue " + - queueName + ".")); + try { + writeLock.lock(); + // check if queue is a valid + List apps = getAppsInQueue(queueName); + if (apps == null) { + String errMsg = "The specified Queue: " + queueName + " doesn't exist"; + LOG.warn(errMsg); + throw new YarnException(errMsg); + } + // generate kill events for each pending/running app + for (ApplicationAttemptId app : apps) { + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL, + "Application killed due to expiry of reservation queue " + + queueName + ".")); + } + } finally { + writeLock.unlock(); } } /** * Process resource update on a node. */ - public synchronized void updateNodeResource(RMNode nm, + public void updateNodeResource(RMNode nm, ResourceOption resourceOption) { - SchedulerNode node = getSchedulerNode(nm.getNodeID()); - Resource newResource = resourceOption.getResource(); - Resource oldResource = node.getTotalResource(); - if(!oldResource.equals(newResource)) { - // Notify NodeLabelsManager about this change - rmContext.getNodeLabelManager().updateNodeResource(nm.getNodeID(), - newResource); - - // Log resource change - LOG.info("Update resource on node: " + node.getNodeName() - + " from: " + oldResource + ", to: " - + newResource); + try { + writeLock.lock(); + SchedulerNode node = getSchedulerNode(nm.getNodeID()); + Resource newResource = resourceOption.getResource(); + Resource oldResource = node.getTotalResource(); + if (!oldResource.equals(newResource)) { + // Notify NodeLabelsManager about this change + rmContext.getNodeLabelManager().updateNodeResource(nm.getNodeID(), + newResource); - nodeTracker.removeNode(nm.getNodeID()); + // Log resource change + LOG.info("Update resource on node: " + node.getNodeName() + " from: " + + oldResource + ", to: " + newResource); - // update resource to node - node.updateTotalResource(newResource); + nodeTracker.removeNode(nm.getNodeID()); - nodeTracker.addNode((N) node); - } else { - // Log resource change - LOG.warn("Update resource on node: " + node.getNodeName() - + " with the same resource: " + newResource); + // update resource to node + node.updateTotalResource(newResource); + + nodeTracker.addNode((N) node); + } else{ + // Log resource change + LOG.warn("Update resource on node: " + node.getNodeName() + + " with the same resource: " + newResource); + } + } finally { + writeLock.unlock(); } } @@ -735,7 +769,7 @@ public abstract class AbstractYarnScheduler } @Override - public synchronized void setClusterMaxPriority(Configuration conf) + public void setClusterMaxPriority(Configuration conf) throws YarnException { try { maxClusterLevelAppPriority = getMaxPriorityFromConf(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index adc3a97c770..d148132ce44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -44,6 +45,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeId; @@ -68,6 +70,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerUpda import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity; + +import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext; import org.apache.hadoop.yarn.state.InvalidStateTransitionException; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; @@ -114,6 +118,9 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { private boolean isAttemptRecovering; protected ResourceUsage attemptResourceUsage = new ResourceUsage(); + /** Resource usage of opportunistic containers. */ + protected ResourceUsage attemptOpportunisticResourceUsage = + new ResourceUsage(); /** Scheduled by a remote scheduler. */ protected ResourceUsage attemptResourceUsageAllocatedRemotely = new ResourceUsage(); @@ -132,6 +139,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { // by NM should not be recovered. private Set pendingRelease = null; + private OpportunisticContainerContext oppContainerContext; + /** * Count how many times the application has been given an opportunity to * schedule a task at each priority. Each time the scheduler asks the @@ -178,7 +187,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { new AppSchedulingInfo(applicationAttemptId, user, queue, activeUsersManager, rmContext.getEpoch(), attemptResourceUsage); this.queue = queue; - this.pendingRelease = new HashSet(); + this.pendingRelease = Collections.newSetFromMap( + new ConcurrentHashMap()); this.attemptId = applicationAttemptId; if (rmContext.getRMApps() != null && rmContext.getRMApps() @@ -199,7 +209,17 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { readLock = lock.readLock(); writeLock = lock.writeLock(); } - + + public void setOpportunisticContainerContext( + OpportunisticContainerContext oppContext) { + this.oppContainerContext = oppContext; + } + + public OpportunisticContainerContext + getOpportunisticContainerContext() { + return this.oppContainerContext; + } + /** * Get the live containers of the application. * @return live containers of the application @@ -331,6 +351,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { try { writeLock.lock(); liveContainers.put(id, rmContainer); + if (rmContainer.getExecutionType() == ExecutionType.OPPORTUNISTIC) { + this.attemptOpportunisticResourceUsage.incUsed( + rmContainer.getAllocatedResource()); + } if (rmContainer.isRemotelyAllocated()) { this.attemptResourceUsageAllocatedRemotely.incUsed( rmContainer.getAllocatedResource()); @@ -344,9 +368,15 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { try { writeLock.lock(); RMContainer rmContainer = liveContainers.remove(containerId); - if (rmContainer != null && rmContainer.isRemotelyAllocated()) { - this.attemptResourceUsageAllocatedRemotely.decUsed( - rmContainer.getAllocatedResource()); + if (rmContainer != null) { + if (rmContainer.getExecutionType() == ExecutionType.OPPORTUNISTIC) { + this.attemptOpportunisticResourceUsage + .decUsed(rmContainer.getAllocatedResource()); + } + if (rmContainer.isRemotelyAllocated()) { + this.attemptResourceUsageAllocatedRemotely + .decUsed(rmContainer.getAllocatedResource()); + } } } finally { writeLock.unlock(); @@ -612,12 +642,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { container.getPriority(), rmContainer.getCreationTime(), this.logAggregationContext, rmContainer.getNodeLabelExpression(), containerType)); - NMToken nmToken = - rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(), - getApplicationAttemptId(), container); - if (nmToken != null) { - updatedNMTokens.add(nmToken); - } + updateNMToken(container); } catch (IllegalArgumentException e) { // DNS might be down, skip returning this container. LOG.error("Error trying to assign container token and NM token to" @@ -635,6 +660,21 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { return container; } + public void updateNMTokens(Collection containers) { + for (Container container : containers) { + updateNMToken(container); + } + } + + private void updateNMToken(Container container) { + NMToken nmToken = + rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(), + getApplicationAttemptId(), container); + if (nmToken != null) { + updatedNMTokens.add(nmToken); + } + } + // Create container token and update NMToken altogether, if either of them fails for // some reason like DNS unavailable, do not return this container and keep it // in the newlyAllocatedContainers waiting to be refetched. @@ -1153,6 +1193,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { // queue's resource usage for specific partition } + public ReentrantReadWriteLock.WriteLock getWriteLock() { + return writeLock; + } + @Override public boolean isRecovering() { return isAttemptRecovering; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 33fe9ad2673..6d00beee8f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -39,7 +39,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -267,8 +266,7 @@ public class CapacityScheduler extends } @Override - public synchronized RMContainerTokenSecretManager - getContainerTokenSecretManager() { + public RMContainerTokenSecretManager getContainerTokenSecretManager() { return this.rmContext.getContainerTokenSecretManager(); } @@ -293,52 +291,62 @@ public class CapacityScheduler extends } @Override - public synchronized RMContext getRMContext() { + public RMContext getRMContext() { return this.rmContext; } @Override - public synchronized void setRMContext(RMContext rmContext) { + public void setRMContext(RMContext rmContext) { this.rmContext = rmContext; } - private synchronized void initScheduler(Configuration configuration) throws + private void initScheduler(Configuration configuration) throws IOException { - this.conf = loadCapacitySchedulerConfiguration(configuration); - validateConf(this.conf); - this.minimumAllocation = this.conf.getMinimumAllocation(); - initMaximumResourceCapability(this.conf.getMaximumAllocation()); - this.calculator = this.conf.getResourceCalculator(); - this.usePortForNodeName = this.conf.getUsePortForNodeName(); - this.applications = new ConcurrentHashMap<>(); - this.labelManager = rmContext.getNodeLabelManager(); - authorizer = YarnAuthorizationProvider.getInstance(yarnConf); - this.activitiesManager = new ActivitiesManager(rmContext); - activitiesManager.init(conf); - initializeQueues(this.conf); - this.isLazyPreemptionEnabled = conf.getLazyPreemptionEnabled(); + try { + writeLock.lock(); + this.conf = loadCapacitySchedulerConfiguration(configuration); + validateConf(this.conf); + this.minimumAllocation = this.conf.getMinimumAllocation(); + initMaximumResourceCapability(this.conf.getMaximumAllocation()); + this.calculator = this.conf.getResourceCalculator(); + this.usePortForNodeName = this.conf.getUsePortForNodeName(); + this.applications = new ConcurrentHashMap<>(); + this.labelManager = rmContext.getNodeLabelManager(); + authorizer = YarnAuthorizationProvider.getInstance(yarnConf); + this.activitiesManager = new ActivitiesManager(rmContext); + activitiesManager.init(conf); + initializeQueues(this.conf); + this.isLazyPreemptionEnabled = conf.getLazyPreemptionEnabled(); - scheduleAsynchronously = this.conf.getScheduleAynschronously(); - asyncScheduleInterval = - this.conf.getLong(ASYNC_SCHEDULER_INTERVAL, - DEFAULT_ASYNC_SCHEDULER_INTERVAL); - if (scheduleAsynchronously) { - asyncSchedulerThread = new AsyncScheduleThread(this); + scheduleAsynchronously = this.conf.getScheduleAynschronously(); + asyncScheduleInterval = this.conf.getLong(ASYNC_SCHEDULER_INTERVAL, + DEFAULT_ASYNC_SCHEDULER_INTERVAL); + if (scheduleAsynchronously) { + asyncSchedulerThread = new AsyncScheduleThread(this); + } + + LOG.info("Initialized CapacityScheduler with " + "calculator=" + + getResourceCalculator().getClass() + ", " + "minimumAllocation=<" + + getMinimumResourceCapability() + ">, " + "maximumAllocation=<" + + getMaximumResourceCapability() + ">, " + "asynchronousScheduling=" + + scheduleAsynchronously + ", " + "asyncScheduleInterval=" + + asyncScheduleInterval + "ms"); + } finally { + writeLock.unlock(); } - - LOG.info("Initialized CapacityScheduler with " + - "calculator=" + getResourceCalculator().getClass() + ", " + - "minimumAllocation=<" + getMinimumResourceCapability() + ">, " + - "maximumAllocation=<" + getMaximumResourceCapability() + ">, " + - "asynchronousScheduling=" + scheduleAsynchronously + ", " + - "asyncScheduleInterval=" + asyncScheduleInterval + "ms"); } - private synchronized void startSchedulerThreads() { - if (scheduleAsynchronously) { - Preconditions.checkNotNull(asyncSchedulerThread, - "asyncSchedulerThread is null"); - asyncSchedulerThread.start(); + private void startSchedulerThreads() { + try { + writeLock.lock(); + activitiesManager.start(); + if (scheduleAsynchronously) { + Preconditions.checkNotNull(asyncSchedulerThread, + "asyncSchedulerThread is null"); + asyncSchedulerThread.start(); + } + } finally { + writeLock.unlock(); } } @@ -352,40 +360,48 @@ public class CapacityScheduler extends @Override public void serviceStart() throws Exception { startSchedulerThreads(); - activitiesManager.start(); super.serviceStart(); } @Override public void serviceStop() throws Exception { - synchronized (this) { + try { + writeLock.lock(); if (scheduleAsynchronously && asyncSchedulerThread != null) { asyncSchedulerThread.interrupt(); asyncSchedulerThread.join(THREAD_JOIN_TIMEOUT_MS); } + } finally { + writeLock.unlock(); } + super.serviceStop(); } @Override - public synchronized void - reinitialize(Configuration conf, RMContext rmContext) throws IOException { - Configuration configuration = new Configuration(conf); - CapacitySchedulerConfiguration oldConf = this.conf; - this.conf = loadCapacitySchedulerConfiguration(configuration); - validateConf(this.conf); + public void reinitialize(Configuration newConf, RMContext rmContext) + throws IOException { try { - LOG.info("Re-initializing queues..."); - refreshMaximumAllocation(this.conf.getMaximumAllocation()); - reinitializeQueues(this.conf); - } catch (Throwable t) { - this.conf = oldConf; - refreshMaximumAllocation(this.conf.getMaximumAllocation()); - throw new IOException("Failed to re-init queues", t); - } + writeLock.lock(); + Configuration configuration = new Configuration(newConf); + CapacitySchedulerConfiguration oldConf = this.conf; + this.conf = loadCapacitySchedulerConfiguration(configuration); + validateConf(this.conf); + try { + LOG.info("Re-initializing queues..."); + refreshMaximumAllocation(this.conf.getMaximumAllocation()); + reinitializeQueues(this.conf); + } catch (Throwable t) { + this.conf = oldConf; + refreshMaximumAllocation(this.conf.getMaximumAllocation()); + throw new IOException("Failed to re-init queues", t); + } - // update lazy preemption - this.isLazyPreemptionEnabled = this.conf.getLazyPreemptionEnabled(); + // update lazy preemption + this.isLazyPreemptionEnabled = this.conf.getLazyPreemptionEnabled(); + } finally { + writeLock.unlock(); + } } long getAsyncScheduleInterval() { @@ -449,10 +465,6 @@ public class CapacityScheduler extends } } - - @Private - public static final String ROOT_QUEUE = - CapacitySchedulerConfiguration.PREFIX + CapacitySchedulerConfiguration.ROOT; static class QueueHook { public CSQueue hook(CSQueue queue) { @@ -462,38 +474,41 @@ public class CapacityScheduler extends private static final QueueHook noop = new QueueHook(); @VisibleForTesting - public synchronized UserGroupMappingPlacementRule + public UserGroupMappingPlacementRule getUserGroupMappingPlacementRule() throws IOException { - boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings(); - LOG.info("Initialized queue mappings, override: " - + overrideWithQueueMappings); + try { + readLock.lock(); + boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings(); + LOG.info( + "Initialized queue mappings, override: " + overrideWithQueueMappings); - // Get new user/group mappings - List newMappings = - conf.getQueueMappings(); - // check if mappings refer to valid queues - for (QueueMapping mapping : newMappings) { - String mappingQueue = mapping.getQueue(); - if (!mappingQueue - .equals(UserGroupMappingPlacementRule.CURRENT_USER_MAPPING) - && !mappingQueue - .equals(UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING)) { - CSQueue queue = queues.get(mappingQueue); - if (queue == null || !(queue instanceof LeafQueue)) { - throw new IOException("mapping contains invalid or non-leaf queue " - + mappingQueue); + // Get new user/group mappings + List newMappings = conf.getQueueMappings(); + // check if mappings refer to valid queues + for (QueueMapping mapping : newMappings) { + String mappingQueue = mapping.getQueue(); + if (!mappingQueue.equals( + UserGroupMappingPlacementRule.CURRENT_USER_MAPPING) && !mappingQueue + .equals(UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING)) { + CSQueue queue = queues.get(mappingQueue); + if (queue == null || !(queue instanceof LeafQueue)) { + throw new IOException( + "mapping contains invalid or non-leaf queue " + mappingQueue); + } } } - } - // initialize groups if mappings are present - if (newMappings.size() > 0) { - Groups groups = new Groups(conf); - return new UserGroupMappingPlacementRule(overrideWithQueueMappings, - newMappings, groups); - } + // initialize groups if mappings are present + if (newMappings.size() > 0) { + Groups groups = new Groups(conf); + return new UserGroupMappingPlacementRule(overrideWithQueueMappings, + newMappings, groups); + } - return null; + return null; + } finally { + readLock.unlock(); + } } private void updatePlacementRules() throws IOException { @@ -526,12 +541,12 @@ public class CapacityScheduler extends } @Lock(CapacityScheduler.class) - private void reinitializeQueues(CapacitySchedulerConfiguration conf) + private void reinitializeQueues(CapacitySchedulerConfiguration newConf) throws IOException { // Parse new queues Map newQueues = new HashMap(); CSQueue newRoot = - parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, + parseQueue(this, newConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, noop); // Ensure all existing queues are still present @@ -693,248 +708,279 @@ public class CapacityScheduler extends return queues.get(queueName); } - private synchronized void addApplicationOnRecovery( + private void addApplicationOnRecovery( ApplicationId applicationId, String queueName, String user, Priority priority) { - CSQueue queue = getQueue(queueName); - if (queue == null) { - //During a restart, this indicates a queue was removed, which is - //not presently supported - if (!YarnConfiguration.shouldRMFailFast(getConfig())) { - this.rmContext.getDispatcher().getEventHandler().handle( - new RMAppEvent(applicationId, RMAppEventType.KILL, - "Application killed on recovery as it was submitted to queue " + - queueName + " which no longer exists after restart.")); - return; - } else { - String queueErrorMsg = "Queue named " + queueName - + " missing during application recovery." - + " Queue removal during recovery is not presently supported by the" - + " capacity scheduler, please restart with all queues configured" - + " which were present before shutdown/restart."; - LOG.fatal(queueErrorMsg); - throw new QueueInvalidException(queueErrorMsg); - } - } - if (!(queue instanceof LeafQueue)) { - // During RM restart, this means leaf queue was converted to a parent - // queue, which is not supported for running apps. - if (!YarnConfiguration.shouldRMFailFast(getConfig())) { - this.rmContext.getDispatcher().getEventHandler().handle( - new RMAppEvent(applicationId, RMAppEventType.KILL, - "Application killed on recovery as it was submitted to queue " + - queueName + " which is no longer a leaf queue after restart.")); - return; - } else { - String queueErrorMsg = "Queue named " + queueName - + " is no longer a leaf queue during application recovery." - + " Changing a leaf queue to a parent queue during recovery is" - + " not presently supported by the capacity scheduler. Please" - + " restart with leaf queues before shutdown/restart continuing" - + " as leaf queues."; - LOG.fatal(queueErrorMsg); - throw new QueueInvalidException(queueErrorMsg); - } - } - // Submit to the queue try { - queue.submitApplication(applicationId, user, queueName); - } catch (AccessControlException ace) { - // Ignore the exception for recovered app as the app was previously - // accepted. - } - queue.getMetrics().submitApp(user); - SchedulerApplication application = - new SchedulerApplication(queue, user, priority); - applications.put(applicationId, application); - LOG.info("Accepted application " + applicationId + " from user: " + user - + ", in queue: " + queueName); - if (LOG.isDebugEnabled()) { - LOG.debug(applicationId + " is recovering. Skip notifying APP_ACCEPTED"); + writeLock.lock(); + CSQueue queue = getQueue(queueName); + if (queue == null) { + //During a restart, this indicates a queue was removed, which is + //not presently supported + if (!YarnConfiguration.shouldRMFailFast(getConfig())) { + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.KILL, + "Application killed on recovery as it was submitted to queue " + + queueName + " which no longer exists after restart.")); + return; + } else{ + String queueErrorMsg = "Queue named " + queueName + + " missing during application recovery." + + " Queue removal during recovery is not presently " + + "supported by the capacity scheduler, please " + + "restart with all queues configured" + + " which were present before shutdown/restart."; + LOG.fatal(queueErrorMsg); + throw new QueueInvalidException(queueErrorMsg); + } + } + if (!(queue instanceof LeafQueue)) { + // During RM restart, this means leaf queue was converted to a parent + // queue, which is not supported for running apps. + if (!YarnConfiguration.shouldRMFailFast(getConfig())) { + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.KILL, + "Application killed on recovery as it was submitted to queue " + + queueName + + " which is no longer a leaf queue after restart.")); + return; + } else{ + String queueErrorMsg = "Queue named " + queueName + + " is no longer a leaf queue during application recovery." + + " Changing a leaf queue to a parent queue during recovery is" + + " not presently supported by the capacity scheduler. Please" + + " restart with leaf queues before shutdown/restart continuing" + + " as leaf queues."; + LOG.fatal(queueErrorMsg); + throw new QueueInvalidException(queueErrorMsg); + } + } + // Submit to the queue + try { + queue.submitApplication(applicationId, user, queueName); + } catch (AccessControlException ace) { + // Ignore the exception for recovered app as the app was previously + // accepted. + } + queue.getMetrics().submitApp(user); + SchedulerApplication application = + new SchedulerApplication(queue, user, priority); + applications.put(applicationId, application); + LOG.info("Accepted application " + applicationId + " from user: " + user + + ", in queue: " + queueName); + if (LOG.isDebugEnabled()) { + LOG.debug( + applicationId + " is recovering. Skip notifying APP_ACCEPTED"); + } + } finally { + writeLock.unlock(); } } - private synchronized void addApplication(ApplicationId applicationId, + private void addApplication(ApplicationId applicationId, String queueName, String user, Priority priority) { - // Sanity checks. - CSQueue queue = getQueue(queueName); - if (queue == null) { - String message = "Application " + applicationId + - " submitted by user " + user + " to unknown queue: " + queueName; - this.rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, - RMAppEventType.APP_REJECTED, message)); - return; - } - if (!(queue instanceof LeafQueue)) { - String message = "Application " + applicationId + - " submitted by user " + user + " to non-leaf queue: " + queueName; - this.rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, - RMAppEventType.APP_REJECTED, message)); - return; - } - // Submit to the queue try { - queue.submitApplication(applicationId, user, queueName); - } catch (AccessControlException ace) { - LOG.info("Failed to submit application " + applicationId + " to queue " - + queueName + " from user " + user, ace); - this.rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, - RMAppEventType.APP_REJECTED, ace.toString())); - return; + writeLock.lock(); + // Sanity checks. + CSQueue queue = getQueue(queueName); + if (queue == null) { + String message = + "Application " + applicationId + " submitted by user " + user + + " to unknown queue: " + queueName; + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED, + message)); + return; + } + if (!(queue instanceof LeafQueue)) { + String message = + "Application " + applicationId + " submitted by user " + user + + " to non-leaf queue: " + queueName; + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED, + message)); + return; + } + // Submit to the queue + try { + queue.submitApplication(applicationId, user, queueName); + } catch (AccessControlException ace) { + LOG.info("Failed to submit application " + applicationId + " to queue " + + queueName + " from user " + user, ace); + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED, + ace.toString())); + return; + } + // update the metrics + queue.getMetrics().submitApp(user); + SchedulerApplication application = + new SchedulerApplication(queue, user, priority); + applications.put(applicationId, application); + LOG.info("Accepted application " + applicationId + " from user: " + user + + ", in queue: " + queueName); + rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED)); + } finally { + writeLock.unlock(); } - // update the metrics - queue.getMetrics().submitApp(user); - SchedulerApplication application = - new SchedulerApplication(queue, user, priority); - applications.put(applicationId, application); - LOG.info("Accepted application " + applicationId + " from user: " + user - + ", in queue: " + queueName); - rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED)); } - private synchronized void addApplicationAttempt( + private void addApplicationAttempt( ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt, boolean isAttemptRecovering) { - SchedulerApplication application = - applications.get(applicationAttemptId.getApplicationId()); - if (application == null) { - LOG.warn("Application " + applicationAttemptId.getApplicationId() + - " cannot be found in scheduler."); - return; - } - CSQueue queue = (CSQueue) application.getQueue(); - - FiCaSchedulerApp attempt = new FiCaSchedulerApp(applicationAttemptId, - application.getUser(), queue, queue.getActiveUsersManager(), rmContext, - application.getPriority(), isAttemptRecovering, activitiesManager); - if (transferStateFromPreviousAttempt) { - attempt.transferStateFromPreviousAttempt( - application.getCurrentAppAttempt()); - } - application.setCurrentAppAttempt(attempt); - - // Update attempt priority to the latest to avoid race condition i.e - // SchedulerApplicationAttempt is created with old priority but it is not - // set to SchedulerApplication#setCurrentAppAttempt. - // Scenario would occur is - // 1. SchdulerApplicationAttempt is created with old priority. - // 2. updateApplicationPriority() updates SchedulerApplication. Since - // currentAttempt is null, it just return. - // 3. ScheduelerApplcationAttempt is set in - // SchedulerApplication#setCurrentAppAttempt. - attempt.setPriority(application.getPriority()); - - queue.submitApplicationAttempt(attempt, application.getUser()); - LOG.info("Added Application Attempt " + applicationAttemptId - + " to scheduler from user " + application.getUser() + " in queue " - + queue.getQueueName()); - if (isAttemptRecovering) { - if (LOG.isDebugEnabled()) { - LOG.debug(applicationAttemptId - + " is recovering. Skipping notifying ATTEMPT_ADDED"); + try { + writeLock.lock(); + SchedulerApplication application = applications.get( + applicationAttemptId.getApplicationId()); + if (application == null) { + LOG.warn("Application " + applicationAttemptId.getApplicationId() + + " cannot be found in scheduler."); + return; } - } else { - rmContext.getDispatcher().getEventHandler().handle( - new RMAppAttemptEvent(applicationAttemptId, - RMAppAttemptEventType.ATTEMPT_ADDED)); - } - } + CSQueue queue = (CSQueue) application.getQueue(); - private synchronized void doneApplication(ApplicationId applicationId, - RMAppState finalState) { - SchedulerApplication application = - applications.get(applicationId); - if (application == null){ - // The AppRemovedSchedulerEvent maybe sent on recovery for completed apps, - // ignore it. - LOG.warn("Couldn't find application " + applicationId); - return; - } - CSQueue queue = (CSQueue) application.getQueue(); - if (!(queue instanceof LeafQueue)) { - LOG.error("Cannot finish application " + "from non-leaf queue: " + FiCaSchedulerApp attempt = new FiCaSchedulerApp(applicationAttemptId, + application.getUser(), queue, queue.getActiveUsersManager(), + rmContext, application.getPriority(), isAttemptRecovering, + activitiesManager); + if (transferStateFromPreviousAttempt) { + attempt.transferStateFromPreviousAttempt( + application.getCurrentAppAttempt()); + } + application.setCurrentAppAttempt(attempt); + + // Update attempt priority to the latest to avoid race condition i.e + // SchedulerApplicationAttempt is created with old priority but it is not + // set to SchedulerApplication#setCurrentAppAttempt. + // Scenario would occur is + // 1. SchdulerApplicationAttempt is created with old priority. + // 2. updateApplicationPriority() updates SchedulerApplication. Since + // currentAttempt is null, it just return. + // 3. ScheduelerApplcationAttempt is set in + // SchedulerApplication#setCurrentAppAttempt. + attempt.setPriority(application.getPriority()); + + queue.submitApplicationAttempt(attempt, application.getUser()); + LOG.info("Added Application Attempt " + applicationAttemptId + + " to scheduler from user " + application.getUser() + " in queue " + queue.getQueueName()); - } else { - queue.finishApplication(applicationId, application.getUser()); + if (isAttemptRecovering) { + if (LOG.isDebugEnabled()) { + LOG.debug(applicationAttemptId + + " is recovering. Skipping notifying ATTEMPT_ADDED"); + } + } else{ + rmContext.getDispatcher().getEventHandler().handle( + new RMAppAttemptEvent(applicationAttemptId, + RMAppAttemptEventType.ATTEMPT_ADDED)); + } + } finally { + writeLock.unlock(); } - application.stop(finalState); - applications.remove(applicationId); } - private synchronized void doneApplicationAttempt( + private void doneApplication(ApplicationId applicationId, + RMAppState finalState) { + try { + writeLock.lock(); + SchedulerApplication application = applications.get( + applicationId); + if (application == null) { + // The AppRemovedSchedulerEvent maybe sent on recovery for completed + // apps, ignore it. + LOG.warn("Couldn't find application " + applicationId); + return; + } + CSQueue queue = (CSQueue) application.getQueue(); + if (!(queue instanceof LeafQueue)) { + LOG.error("Cannot finish application " + "from non-leaf queue: " + queue + .getQueueName()); + } else{ + queue.finishApplication(applicationId, application.getUser()); + } + application.stop(finalState); + applications.remove(applicationId); + } finally { + writeLock.unlock(); + } + } + + private void doneApplicationAttempt( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) { - LOG.info("Application Attempt " + applicationAttemptId + " is done." + - " finalState=" + rmAppAttemptFinalState); - - FiCaSchedulerApp attempt = getApplicationAttempt(applicationAttemptId); - SchedulerApplication application = - applications.get(applicationAttemptId.getApplicationId()); + try { + writeLock.lock(); + LOG.info("Application Attempt " + applicationAttemptId + " is done." + + " finalState=" + rmAppAttemptFinalState); - if (application == null || attempt == null) { - LOG.info("Unknown application " + applicationAttemptId + " has completed!"); - return; - } + FiCaSchedulerApp attempt = getApplicationAttempt(applicationAttemptId); + SchedulerApplication application = applications.get( + applicationAttemptId.getApplicationId()); - // Release all the allocated, acquired, running containers - for (RMContainer rmContainer : attempt.getLiveContainers()) { - if (keepContainers - && rmContainer.getState().equals(RMContainerState.RUNNING)) { - // do not kill the running container in the case of work-preserving AM - // restart. - LOG.info("Skip killing " + rmContainer.getContainerId()); - continue; + if (application == null || attempt == null) { + LOG.info( + "Unknown application " + applicationAttemptId + " has completed!"); + return; } - super.completedContainer( - rmContainer, - SchedulerUtils.createAbnormalContainerStatus( - rmContainer.getContainerId(), SchedulerUtils.COMPLETED_APPLICATION), - RMContainerEventType.KILL); - } - // Release all reserved containers - for (RMContainer rmContainer : attempt.getReservedContainers()) { - super.completedContainer( - rmContainer, - SchedulerUtils.createAbnormalContainerStatus( - rmContainer.getContainerId(), "Application Complete"), - RMContainerEventType.KILL); - } + // Release all the allocated, acquired, running containers + for (RMContainer rmContainer : attempt.getLiveContainers()) { + if (keepContainers && rmContainer.getState().equals( + RMContainerState.RUNNING)) { + // do not kill the running container in the case of work-preserving AM + // restart. + LOG.info("Skip killing " + rmContainer.getContainerId()); + continue; + } + super.completedContainer(rmContainer, SchedulerUtils + .createAbnormalContainerStatus(rmContainer.getContainerId(), + SchedulerUtils.COMPLETED_APPLICATION), + RMContainerEventType.KILL); + } - // Clean up pending requests, metrics etc. - attempt.stop(rmAppAttemptFinalState); + // Release all reserved containers + for (RMContainer rmContainer : attempt.getReservedContainers()) { + super.completedContainer(rmContainer, SchedulerUtils + .createAbnormalContainerStatus(rmContainer.getContainerId(), + "Application Complete"), RMContainerEventType.KILL); + } - // Inform the queue - String queueName = attempt.getQueue().getQueueName(); - CSQueue queue = queues.get(queueName); - if (!(queue instanceof LeafQueue)) { - LOG.error("Cannot finish application " + "from non-leaf queue: " - + queueName); - } else { - queue.finishApplicationAttempt(attempt, queue.getQueueName()); + // Clean up pending requests, metrics etc. + attempt.stop(rmAppAttemptFinalState); + + // Inform the queue + String queueName = attempt.getQueue().getQueueName(); + CSQueue queue = queues.get(queueName); + if (!(queue instanceof LeafQueue)) { + LOG.error( + "Cannot finish application " + "from non-leaf queue: " + queueName); + } else{ + queue.finishApplicationAttempt(attempt, queue.getQueueName()); + } + } finally { + writeLock.unlock(); } } - // It is crucial to acquire leaf queue lock first to prevent: - // 1. Race condition when calculating the delta resource in - // SchedContainerChangeRequest - // 2. Deadlock with the scheduling thread. private LeafQueue updateIncreaseRequests( - List increaseRequests, - FiCaSchedulerApp app) { + List increaseRequests, FiCaSchedulerApp app) { if (null == increaseRequests || increaseRequests.isEmpty()) { return null; } + // Pre-process increase requests List schedIncreaseRequests = createSchedContainerChangeRequests(increaseRequests, true); LeafQueue leafQueue = (LeafQueue) app.getQueue(); - synchronized(leafQueue) { + + try { + /* + * Acquire application's lock here to make sure application won't + * finish when updateIncreaseRequest is called. + */ + app.getWriteLock().lock(); // make sure we aren't stopping/removing the application // when the allocate comes in if (app.isStopped()) { @@ -944,8 +990,12 @@ public class CapacityScheduler extends if (app.updateIncreaseRequests(schedIncreaseRequests)) { return leafQueue; } - return null; + } finally { + app.getWriteLock().unlock(); } + + + return null; } @Override @@ -955,7 +1005,6 @@ public class CapacityScheduler extends List blacklistAdditions, List blacklistRemovals, List increaseRequests, List decreaseRequests) { - FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId); if (application == null) { return EMPTY_ALLOCATION; @@ -965,42 +1014,43 @@ public class CapacityScheduler extends releaseContainers(release, application); // update increase requests - LeafQueue updateDemandForQueue = - updateIncreaseRequests(increaseRequests, application); + LeafQueue updateDemandForQueue = updateIncreaseRequests(increaseRequests, + application); // Decrease containers decreaseContainers(decreaseRequests, application); // Sanity check for new allocation requests - SchedulerUtils.normalizeRequests( - ask, getResourceCalculator(), getClusterResource(), - getMinimumResourceCapability(), getMaximumResourceCapability()); + SchedulerUtils.normalizeRequests(ask, getResourceCalculator(), + getClusterResource(), getMinimumResourceCapability(), + getMaximumResourceCapability()); Allocation allocation; - synchronized (application) { - - // make sure we aren't stopping/removing the application - // when the allocate comes in + // make sure we aren't stopping/removing the application + // when the allocate comes in + try { + application.getWriteLock().lock(); if (application.isStopped()) { return EMPTY_ALLOCATION; } // Process resource requests if (!ask.isEmpty()) { - if(LOG.isDebugEnabled()) { - LOG.debug("allocate: pre-update " + applicationAttemptId + - " ask size =" + ask.size()); + if (LOG.isDebugEnabled()) { + LOG.debug( + "allocate: pre-update " + applicationAttemptId + " ask size =" + + ask.size()); application.showRequests(); } // Update application requests - if (application.updateResourceRequests(ask) - && (updateDemandForQueue == null)) { + if (application.updateResourceRequests(ask) && (updateDemandForQueue + == null)) { updateDemandForQueue = (LeafQueue) application.getQueue(); } - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("allocate: post-update"); application.showRequests(); } @@ -1010,6 +1060,8 @@ public class CapacityScheduler extends allocation = application.getAllocation(getResourceCalculator(), getClusterResource(), getMinimumResourceCapability()); + } finally { + application.getWriteLock().unlock(); } if (updateDemandForQueue != null && !application @@ -1018,7 +1070,6 @@ public class CapacityScheduler extends } return allocation; - } @Override @@ -1048,142 +1099,159 @@ public class CapacityScheduler extends return root.getQueueUserAclInfo(user); } - private synchronized void nodeUpdate(RMNode nm) { - if (LOG.isDebugEnabled()) { - LOG.debug("nodeUpdate: " + nm + - " clusterResources: " + getClusterResource()); - } + private void nodeUpdate(RMNode nm) { + try { + writeLock.lock(); + if (LOG.isDebugEnabled()) { + LOG.debug( + "nodeUpdate: " + nm + " clusterResources: " + getClusterResource()); + } - Resource releaseResources = Resource.newInstance(0, 0); + Resource releaseResources = Resource.newInstance(0, 0); - FiCaSchedulerNode node = getNode(nm.getNodeID()); - - List containerInfoList = nm.pullContainerUpdates(); - List newlyLaunchedContainers = new ArrayList(); - List completedContainers = new ArrayList(); - for(UpdatedContainerInfo containerInfo : containerInfoList) { - newlyLaunchedContainers.addAll(containerInfo.getNewlyLaunchedContainers()); - completedContainers.addAll(containerInfo.getCompletedContainers()); - } - - // Processing the newly launched containers - for (ContainerStatus launchedContainer : newlyLaunchedContainers) { - containerLaunchedOnNode(launchedContainer.getContainerId(), node); - } - - // Processing the newly increased containers - List newlyIncreasedContainers = - nm.pullNewlyIncreasedContainers(); - for (Container container : newlyIncreasedContainers) { - containerIncreasedOnNode(container.getId(), node, container); - } + FiCaSchedulerNode node = getNode(nm.getNodeID()); - // Process completed containers - int releasedContainers = 0; - for (ContainerStatus completedContainer : completedContainers) { - ContainerId containerId = completedContainer.getContainerId(); - RMContainer container = getRMContainer(containerId); - super.completedContainer(container, completedContainer, - RMContainerEventType.FINISHED); - if (container != null) { - releasedContainers++; - Resource rs = container.getAllocatedResource(); - if (rs != null) { - Resources.addTo(releaseResources, rs); - } - rs = container.getReservedResource(); - if (rs != null) { - Resources.addTo(releaseResources, rs); + List containerInfoList = nm.pullContainerUpdates(); + List newlyLaunchedContainers = + new ArrayList(); + List completedContainers = + new ArrayList(); + for (UpdatedContainerInfo containerInfo : containerInfoList) { + newlyLaunchedContainers.addAll( + containerInfo.getNewlyLaunchedContainers()); + completedContainers.addAll(containerInfo.getCompletedContainers()); + } + + // Processing the newly launched containers + for (ContainerStatus launchedContainer : newlyLaunchedContainers) { + containerLaunchedOnNode(launchedContainer.getContainerId(), node); + } + + // Processing the newly increased containers + List newlyIncreasedContainers = + nm.pullNewlyIncreasedContainers(); + for (Container container : newlyIncreasedContainers) { + containerIncreasedOnNode(container.getId(), node, container); + } + + // Process completed containers + int releasedContainers = 0; + for (ContainerStatus completedContainer : completedContainers) { + ContainerId containerId = completedContainer.getContainerId(); + RMContainer container = getRMContainer(containerId); + super.completedContainer(container, completedContainer, + RMContainerEventType.FINISHED); + if (container != null) { + releasedContainers++; + Resource rs = container.getAllocatedResource(); + if (rs != null) { + Resources.addTo(releaseResources, rs); + } + rs = container.getReservedResource(); + if (rs != null) { + Resources.addTo(releaseResources, rs); + } } } - } - // If the node is decommissioning, send an update to have the total - // resource equal to the used resource, so no available resource to - // schedule. - // TODO: Fix possible race-condition when request comes in before - // update is propagated - if (nm.getState() == NodeState.DECOMMISSIONING) { - this.rmContext - .getDispatcher() - .getEventHandler() - .handle( - new RMNodeResourceUpdateEvent(nm.getNodeID(), ResourceOption - .newInstance(getSchedulerNode(nm.getNodeID()) - .getAllocatedResource(), 0))); - } - schedulerHealth.updateSchedulerReleaseDetails(lastNodeUpdateTime, - releaseResources); - schedulerHealth.updateSchedulerReleaseCounts(releasedContainers); + // If the node is decommissioning, send an update to have the total + // resource equal to the used resource, so no available resource to + // schedule. + // TODO: Fix possible race-condition when request comes in before + // update is propagated + if (nm.getState() == NodeState.DECOMMISSIONING) { + this.rmContext.getDispatcher().getEventHandler().handle( + new RMNodeResourceUpdateEvent(nm.getNodeID(), ResourceOption + .newInstance( + getSchedulerNode(nm.getNodeID()).getAllocatedResource(), + 0))); + } + schedulerHealth.updateSchedulerReleaseDetails(lastNodeUpdateTime, + releaseResources); + schedulerHealth.updateSchedulerReleaseCounts(releasedContainers); - // Updating node resource utilization - node.setAggregatedContainersUtilization( - nm.getAggregatedContainersUtilization()); - node.setNodeUtilization(nm.getNodeUtilization()); + // Updating node resource utilization + node.setAggregatedContainersUtilization( + nm.getAggregatedContainersUtilization()); + node.setNodeUtilization(nm.getNodeUtilization()); - // Now node data structures are upto date and ready for scheduling. - if(LOG.isDebugEnabled()) { - LOG.debug("Node being looked for scheduling " + nm + - " availableResource: " + node.getUnallocatedResource()); + // Now node data structures are upto date and ready for scheduling. + if (LOG.isDebugEnabled()) { + LOG.debug( + "Node being looked for scheduling " + nm + " availableResource: " + + node.getUnallocatedResource()); + } + } finally { + writeLock.unlock(); } } /** * Process resource update on a node. */ - private synchronized void updateNodeAndQueueResource(RMNode nm, + private void updateNodeAndQueueResource(RMNode nm, ResourceOption resourceOption) { - updateNodeResource(nm, resourceOption); - Resource clusterResource = getClusterResource(); - root.updateClusterResource(clusterResource, new ResourceLimits( - clusterResource)); + try { + writeLock.lock(); + updateNodeResource(nm, resourceOption); + Resource clusterResource = getClusterResource(); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + } finally { + writeLock.unlock(); + } } /** * Process node labels update on a node. */ - private synchronized void updateLabelsOnNode(NodeId nodeId, + private void updateLabelsOnNode(NodeId nodeId, Set newLabels) { - FiCaSchedulerNode node = nodeTracker.getNode(nodeId); - if (null == node) { - return; - } - - // Get new partition, we have only one partition per node - String newPartition; - if (newLabels.isEmpty()) { - newPartition = RMNodeLabelsManager.NO_LABEL; - } else { - newPartition = newLabels.iterator().next(); - } - - // old partition as well - String oldPartition = node.getPartition(); - - // Update resources of these containers - for (RMContainer rmContainer : node.getCopiedListOfRunningContainers()) { - FiCaSchedulerApp application = - getApplicationAttempt(rmContainer.getApplicationAttemptId()); - if (null != application) { - application.nodePartitionUpdated(rmContainer, oldPartition, - newPartition); - } else { - LOG.warn("There's something wrong, some RMContainers running on" - + " a node, but we cannot find SchedulerApplicationAttempt for it. Node=" - + node.getNodeID() + " applicationAttemptId=" - + rmContainer.getApplicationAttemptId()); - continue; + try { + writeLock.lock(); + FiCaSchedulerNode node = nodeTracker.getNode(nodeId); + if (null == node) { + return; } + + // Get new partition, we have only one partition per node + String newPartition; + if (newLabels.isEmpty()) { + newPartition = RMNodeLabelsManager.NO_LABEL; + } else{ + newPartition = newLabels.iterator().next(); + } + + // old partition as well + String oldPartition = node.getPartition(); + + // Update resources of these containers + for (RMContainer rmContainer : node.getCopiedListOfRunningContainers()) { + FiCaSchedulerApp application = getApplicationAttempt( + rmContainer.getApplicationAttemptId()); + if (null != application) { + application.nodePartitionUpdated(rmContainer, oldPartition, + newPartition); + } else{ + LOG.warn("There's something wrong, some RMContainers running on" + + " a node, but we cannot find SchedulerApplicationAttempt " + + "for it. Node=" + node.getNodeID() + " applicationAttemptId=" + + rmContainer.getApplicationAttemptId()); + continue; + } + } + + // Unreserve container on this node + RMContainer reservedContainer = node.getReservedContainer(); + if (null != reservedContainer) { + killReservedContainer(reservedContainer); + } + + // Update node labels after we've done this + node.updateLabels(newLabels); + } finally { + writeLock.unlock(); } - - // Unreserve container on this node - RMContainer reservedContainer = node.getReservedContainer(); - if (null != reservedContainer) { - killReservedContainer(reservedContainer); - } - - // Update node labels after we've done this - node.updateLabels(newLabels); } private void updateSchedulerHealth(long now, FiCaSchedulerNode node, @@ -1218,134 +1286,134 @@ public class CapacityScheduler extends } @VisibleForTesting - public synchronized void allocateContainersToNode(FiCaSchedulerNode node) { - if (rmContext.isWorkPreservingRecoveryEnabled() - && !rmContext.isSchedulerReadyForAllocatingContainers()) { - return; - } - - if (!nodeTracker.exists(node.getNodeID())) { - LOG.info("Skipping scheduling as the node " + node.getNodeID() + - " has been removed"); - return; - } - - // reset allocation and reservation stats before we start doing any work - updateSchedulerHealth(lastNodeUpdateTime, node, - new CSAssignment(Resources.none(), NodeType.NODE_LOCAL)); - - CSAssignment assignment; - - // Assign new containers... - // 1. Check for reserved applications - // 2. Schedule if there are no reservations - - RMContainer reservedContainer = node.getReservedContainer(); - if (reservedContainer != null) { - - FiCaSchedulerApp reservedApplication = - getCurrentAttemptForContainer(reservedContainer.getContainerId()); - - // Try to fulfill the reservation - LOG.info("Trying to fulfill reservation for application " - + reservedApplication.getApplicationId() + " on node: " - + node.getNodeID()); - - LeafQueue queue = ((LeafQueue) reservedApplication.getQueue()); - assignment = - queue.assignContainers( - getClusterResource(), - node, - // TODO, now we only consider limits for parent for non-labeled - // resources, should consider labeled resources as well. - new ResourceLimits(labelManager.getResourceByLabel( - RMNodeLabelsManager.NO_LABEL, getClusterResource())), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); - if (assignment.isFulfilledReservation()) { - CSAssignment tmp = - new CSAssignment(reservedContainer.getReservedResource(), - assignment.getType()); - Resources.addTo(assignment.getAssignmentInformation().getAllocated(), - reservedContainer.getReservedResource()); - tmp.getAssignmentInformation().addAllocationDetails( - reservedContainer.getContainerId(), queue.getQueuePath()); - tmp.getAssignmentInformation().incrAllocations(); - updateSchedulerHealth(lastNodeUpdateTime, node, tmp); - schedulerHealth.updateSchedulerFulfilledReservationCounts(1); - - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - queue.getParent().getQueueName(), queue.getQueueName(), - ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); - ActivitiesLogger.NODE.finishAllocatedNodeAllocation(activitiesManager, - node, reservedContainer.getContainerId(), - AllocationState.ALLOCATED_FROM_RESERVED); - } else { - ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - queue.getParent().getQueueName(), queue.getQueueName(), - ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); - ActivitiesLogger.NODE.finishAllocatedNodeAllocation(activitiesManager, - node, reservedContainer.getContainerId(), AllocationState.SKIPPED); + public void allocateContainersToNode(FiCaSchedulerNode node) { + try { + writeLock.lock(); + if (rmContext.isWorkPreservingRecoveryEnabled() && !rmContext + .isSchedulerReadyForAllocatingContainers()) { + return; } - } - // Try to schedule more if there are no reservations to fulfill - if (node.getReservedContainer() == null) { - if (calculator.computeAvailableContainers(Resources - .add(node.getUnallocatedResource(), node.getTotalKillableResources()), - minimumAllocation) > 0) { + if (!nodeTracker.exists(node.getNodeID())) { + LOG.info("Skipping scheduling as the node " + node.getNodeID() + + " has been removed"); + return; + } - if (LOG.isDebugEnabled()) { - LOG.debug("Trying to schedule on node: " + node.getNodeName() + - ", available: " + node.getUnallocatedResource()); - } + // reset allocation and reservation stats before we start doing any work + updateSchedulerHealth(lastNodeUpdateTime, node, + new CSAssignment(Resources.none(), NodeType.NODE_LOCAL)); - assignment = root.assignContainers( - getClusterResource(), - node, - new ResourceLimits(labelManager.getResourceByLabel( - node.getPartition(), getClusterResource())), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); - if (Resources.greaterThan(calculator, getClusterResource(), - assignment.getResource(), Resources.none())) { - updateSchedulerHealth(lastNodeUpdateTime, node, assignment); - return; - } - - // Only do non-exclusive allocation when node has node-labels. - if (StringUtils.equals(node.getPartition(), - RMNodeLabelsManager.NO_LABEL)) { - return; - } - - // Only do non-exclusive allocation when the node-label supports that - try { - if (rmContext.getNodeLabelManager().isExclusiveNodeLabel( - node.getPartition())) { - return; - } - } catch (IOException e) { - LOG.warn("Exception when trying to get exclusivity of node label=" - + node.getPartition(), e); - return; - } - - // Try to use NON_EXCLUSIVE - assignment = root.assignContainers( - getClusterResource(), - node, + CSAssignment assignment; + + // Assign new containers... + // 1. Check for reserved applications + // 2. Schedule if there are no reservations + + RMContainer reservedContainer = node.getReservedContainer(); + if (reservedContainer != null) { + + FiCaSchedulerApp reservedApplication = getCurrentAttemptForContainer( + reservedContainer.getContainerId()); + + // Try to fulfill the reservation + LOG.info("Trying to fulfill reservation for application " + + reservedApplication.getApplicationId() + " on node: " + node + .getNodeID()); + + LeafQueue queue = ((LeafQueue) reservedApplication.getQueue()); + assignment = queue.assignContainers(getClusterResource(), node, // TODO, now we only consider limits for parent for non-labeled // resources, should consider labeled resources as well. - new ResourceLimits(labelManager.getResourceByLabel( - RMNodeLabelsManager.NO_LABEL, getClusterResource())), - SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY); - updateSchedulerHealth(lastNodeUpdateTime, node, assignment); + new ResourceLimits(labelManager + .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, + getClusterResource())), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + if (assignment.isFulfilledReservation()) { + CSAssignment tmp = new CSAssignment( + reservedContainer.getReservedResource(), assignment.getType()); + Resources.addTo(assignment.getAssignmentInformation().getAllocated(), + reservedContainer.getReservedResource()); + tmp.getAssignmentInformation().addAllocationDetails( + reservedContainer.getContainerId(), queue.getQueuePath()); + tmp.getAssignmentInformation().incrAllocations(); + updateSchedulerHealth(lastNodeUpdateTime, node, tmp); + schedulerHealth.updateSchedulerFulfilledReservationCounts(1); + + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, + queue.getParent().getQueueName(), queue.getQueueName(), + ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); + ActivitiesLogger.NODE.finishAllocatedNodeAllocation(activitiesManager, + node, reservedContainer.getContainerId(), + AllocationState.ALLOCATED_FROM_RESERVED); + } else{ + ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, + queue.getParent().getQueueName(), queue.getQueueName(), + ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); + ActivitiesLogger.NODE.finishAllocatedNodeAllocation(activitiesManager, + node, reservedContainer.getContainerId(), + AllocationState.SKIPPED); + } } - } else { - LOG.info("Skipping scheduling since node " - + node.getNodeID() - + " is reserved by application " - + node.getReservedContainer().getContainerId() - .getApplicationAttemptId()); + + // Try to schedule more if there are no reservations to fulfill + if (node.getReservedContainer() == null) { + if (calculator.computeAvailableContainers(Resources + .add(node.getUnallocatedResource(), + node.getTotalKillableResources()), minimumAllocation) > 0) { + + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to schedule on node: " + node.getNodeName() + + ", available: " + node.getUnallocatedResource()); + } + + assignment = root.assignContainers(getClusterResource(), node, + new ResourceLimits(labelManager + .getResourceByLabel(node.getPartition(), + getClusterResource())), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); + if (Resources.greaterThan(calculator, getClusterResource(), + assignment.getResource(), Resources.none())) { + updateSchedulerHealth(lastNodeUpdateTime, node, assignment); + return; + } + + // Only do non-exclusive allocation when node has node-labels. + if (StringUtils.equals(node.getPartition(), + RMNodeLabelsManager.NO_LABEL)) { + return; + } + + // Only do non-exclusive allocation when the node-label supports that + try { + if (rmContext.getNodeLabelManager().isExclusiveNodeLabel( + node.getPartition())) { + return; + } + } catch (IOException e) { + LOG.warn( + "Exception when trying to get exclusivity of node label=" + node + .getPartition(), e); + return; + } + + // Try to use NON_EXCLUSIVE + assignment = root.assignContainers(getClusterResource(), node, + // TODO, now we only consider limits for parent for non-labeled + // resources, should consider labeled resources as well. + new ResourceLimits(labelManager + .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, + getClusterResource())), + SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY); + updateSchedulerHealth(lastNodeUpdateTime, node, assignment); + } + } else{ + LOG.info("Skipping scheduling since node " + node.getNodeID() + + " is reserved by application " + node.getReservedContainer() + .getContainerId().getApplicationAttemptId()); + } + } finally { + writeLock.unlock(); } } @@ -1498,100 +1566,108 @@ public class CapacityScheduler extends } } - private synchronized void addNode(RMNode nodeManager) { - FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager, - usePortForNodeName, nodeManager.getNodeLabels()); - nodeTracker.addNode(schedulerNode); + private void addNode(RMNode nodeManager) { + try { + writeLock.lock(); + FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager, + usePortForNodeName, nodeManager.getNodeLabels()); + nodeTracker.addNode(schedulerNode); - // update this node to node label manager - if (labelManager != null) { - labelManager.activateNode(nodeManager.getNodeID(), - schedulerNode.getTotalResource()); - } + // update this node to node label manager + if (labelManager != null) { + labelManager.activateNode(nodeManager.getNodeID(), + schedulerNode.getTotalResource()); + } - Resource clusterResource = getClusterResource(); - root.updateClusterResource(clusterResource, new ResourceLimits( - clusterResource)); + Resource clusterResource = getClusterResource(); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); - LOG.info("Added node " + nodeManager.getNodeAddress() + - " clusterResource: " + clusterResource); + LOG.info( + "Added node " + nodeManager.getNodeAddress() + " clusterResource: " + + clusterResource); - if (scheduleAsynchronously && getNumClusterNodes() == 1) { - asyncSchedulerThread.beginSchedule(); + if (scheduleAsynchronously && getNumClusterNodes() == 1) { + asyncSchedulerThread.beginSchedule(); + } + } finally { + writeLock.unlock(); } } - private synchronized void removeNode(RMNode nodeInfo) { - // update this node to node label manager - if (labelManager != null) { - labelManager.deactivateNode(nodeInfo.getNodeID()); - } + private void removeNode(RMNode nodeInfo) { + try { + writeLock.lock(); + // update this node to node label manager + if (labelManager != null) { + labelManager.deactivateNode(nodeInfo.getNodeID()); + } - NodeId nodeId = nodeInfo.getNodeID(); - FiCaSchedulerNode node = nodeTracker.getNode(nodeId); - if (node == null) { - LOG.error("Attempting to remove non-existent node " + nodeId); - return; - } + NodeId nodeId = nodeInfo.getNodeID(); + FiCaSchedulerNode node = nodeTracker.getNode(nodeId); + if (node == null) { + LOG.error("Attempting to remove non-existent node " + nodeId); + return; + } - // Remove running containers - List runningContainers = node.getCopiedListOfRunningContainers(); - for (RMContainer container : runningContainers) { - super.completedContainer(container, - SchedulerUtils.createAbnormalContainerStatus( - container.getContainerId(), - SchedulerUtils.LOST_CONTAINER), - RMContainerEventType.KILL); - } - - // Remove reservations, if any - RMContainer reservedContainer = node.getReservedContainer(); - if (reservedContainer != null) { - super.completedContainer(reservedContainer, - SchedulerUtils.createAbnormalContainerStatus( - reservedContainer.getContainerId(), - SchedulerUtils.LOST_CONTAINER), - RMContainerEventType.KILL); - } + // Remove running containers + List runningContainers = + node.getCopiedListOfRunningContainers(); + for (RMContainer container : runningContainers) { + super.completedContainer(container, SchedulerUtils + .createAbnormalContainerStatus(container.getContainerId(), + SchedulerUtils.LOST_CONTAINER), RMContainerEventType.KILL); + } - nodeTracker.removeNode(nodeId); - Resource clusterResource = getClusterResource(); - root.updateClusterResource(clusterResource, new ResourceLimits( - clusterResource)); - int numNodes = nodeTracker.nodeCount(); + // Remove reservations, if any + RMContainer reservedContainer = node.getReservedContainer(); + if (reservedContainer != null) { + super.completedContainer(reservedContainer, SchedulerUtils + .createAbnormalContainerStatus(reservedContainer.getContainerId(), + SchedulerUtils.LOST_CONTAINER), RMContainerEventType.KILL); + } - if (scheduleAsynchronously && numNodes == 0) { - asyncSchedulerThread.suspendSchedule(); + nodeTracker.removeNode(nodeId); + Resource clusterResource = getClusterResource(); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + int numNodes = nodeTracker.nodeCount(); + + if (scheduleAsynchronously && numNodes == 0) { + asyncSchedulerThread.suspendSchedule(); + } + + LOG.info( + "Removed node " + nodeInfo.getNodeAddress() + " clusterResource: " + + getClusterResource()); + } finally { + writeLock.unlock(); } - - LOG.info("Removed node " + nodeInfo.getNodeAddress() + - " clusterResource: " + getClusterResource()); } private void rollbackContainerResource( ContainerId containerId) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { - LOG.info("Cannot rollback resource for container " + containerId + - ". The container does not exist."); + LOG.info("Cannot rollback resource for container " + containerId + + ". The container does not exist."); return; } FiCaSchedulerApp application = getCurrentAttemptForContainer(containerId); if (application == null) { - LOG.info("Cannot rollback resource for container " + containerId + - ". The application that the container belongs to does not exist."); + LOG.info("Cannot rollback resource for container " + containerId + + ". The application that the container " + + "belongs to does not exist."); return; } LOG.info("Roll back resource for container " + containerId); - LeafQueue leafQueue = (LeafQueue) application.getQueue(); - synchronized(leafQueue) { - SchedulerNode schedulerNode = - getSchedulerNode(rmContainer.getAllocatedNode()); - SchedContainerChangeRequest decreaseRequest = - new SchedContainerChangeRequest(this.rmContext, schedulerNode, - rmContainer, rmContainer.getLastConfirmedResource()); - decreaseContainer(decreaseRequest, application); - } + + SchedulerNode schedulerNode = getSchedulerNode( + rmContainer.getAllocatedNode()); + SchedContainerChangeRequest decreaseRequest = + new SchedContainerChangeRequest(this.rmContext, schedulerNode, + rmContainer, rmContainer.getLastConfirmedResource()); + decreaseContainer(decreaseRequest, application); } @Override @@ -1600,23 +1676,29 @@ public class CapacityScheduler extends RMContainerEventType event) { Container container = rmContainer.getContainer(); ContainerId containerId = container.getId(); - + // Get the application for the finished container - FiCaSchedulerApp application = - getCurrentAttemptForContainer(container.getId()); + FiCaSchedulerApp application = getCurrentAttemptForContainer( + container.getId()); ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId(); if (application == null) { - LOG.info("Container " + container + " of" + " finished application " - + appId + " completed with event " + event); + LOG.info( + "Container " + container + " of" + " finished application " + appId + + " completed with event " + event); return; } - + // Get the node on which the container was allocated FiCaSchedulerNode node = getNode(container.getNodeId()); - + if (null == node) { + LOG.info("Container " + container + " of" + " removed node " + container + .getNodeId() + " completed with event " + event); + return; + } + // Inform the queue - LeafQueue queue = (LeafQueue)application.getQueue(); + LeafQueue queue = (LeafQueue) application.getQueue(); queue.completedContainer(getClusterResource(), application, node, rmContainer, containerStatus, event, null, true); } @@ -1627,19 +1709,19 @@ public class CapacityScheduler extends RMContainer rmContainer = decreaseRequest.getRMContainer(); // Check container status before doing decrease if (rmContainer.getState() != RMContainerState.RUNNING) { - LOG.info("Trying to decrease a container not in RUNNING state, container=" - + rmContainer + " state=" + rmContainer.getState().name()); + LOG.info( + "Trying to decrease a container not in RUNNING state, container=" + + rmContainer + " state=" + rmContainer.getState().name()); return; } - FiCaSchedulerApp app = (FiCaSchedulerApp)attempt; + FiCaSchedulerApp app = (FiCaSchedulerApp) attempt; LeafQueue queue = (LeafQueue) attempt.getQueue(); try { queue.decreaseContainer(getClusterResource(), decreaseRequest, app); // Notify RMNode that the container can be pulled by NodeManager in the // next heartbeat - this.rmContext.getDispatcher().getEventHandler() - .handle(new RMNodeDecreaseContainerEvent( - decreaseRequest.getNodeId(), + this.rmContext.getDispatcher().getEventHandler().handle( + new RMNodeDecreaseContainerEvent(decreaseRequest.getNodeId(), Collections.singletonList(rmContainer.getContainer()))); } catch (InvalidResourceRequestException e) { LOG.warn("Error happens when checking decrease request, Ignoring.." @@ -1700,70 +1782,81 @@ public class CapacityScheduler extends } } - public synchronized void markContainerForKillable( + public void markContainerForKillable( RMContainer killableContainer) { - if (LOG.isDebugEnabled()) { - LOG.debug(SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE + ": container" - + killableContainer.toString()); - } + try { + writeLock.lock(); + if (LOG.isDebugEnabled()) { + LOG.debug(SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE + ": container" + + killableContainer.toString()); + } + + if (!isLazyPreemptionEnabled) { + super.completedContainer(killableContainer, SchedulerUtils + .createPreemptedContainerStatus(killableContainer.getContainerId(), + SchedulerUtils.PREEMPTED_CONTAINER), RMContainerEventType.KILL); + } else{ + FiCaSchedulerNode node = (FiCaSchedulerNode) getSchedulerNode( + killableContainer.getAllocatedNode()); + + FiCaSchedulerApp application = getCurrentAttemptForContainer( + killableContainer.getContainerId()); + + node.markContainerToKillable(killableContainer.getContainerId()); + + // notify PreemptionManager + // Get the application for the finished container + if (null != application) { + String leafQueueName = application.getCSLeafQueue().getQueueName(); + getPreemptionManager().addKillableContainer( + new KillableContainer(killableContainer, node.getPartition(), + leafQueueName)); + } + } + } finally { + writeLock.unlock(); + } + } + + private void markContainerForNonKillable( + RMContainer nonKillableContainer) { + try { + writeLock.lock(); + if (LOG.isDebugEnabled()) { + LOG.debug( + SchedulerEventType.MARK_CONTAINER_FOR_NONKILLABLE + ": container" + + nonKillableContainer.toString()); + } - if (!isLazyPreemptionEnabled) { - super.completedContainer(killableContainer, SchedulerUtils - .createPreemptedContainerStatus(killableContainer.getContainerId(), - SchedulerUtils.PREEMPTED_CONTAINER), RMContainerEventType.KILL); - } else { FiCaSchedulerNode node = (FiCaSchedulerNode) getSchedulerNode( - killableContainer.getAllocatedNode()); + nonKillableContainer.getAllocatedNode()); FiCaSchedulerApp application = getCurrentAttemptForContainer( - killableContainer.getContainerId()); + nonKillableContainer.getContainerId()); - node.markContainerToKillable(killableContainer.getContainerId()); + node.markContainerToNonKillable(nonKillableContainer.getContainerId()); // notify PreemptionManager // Get the application for the finished container if (null != application) { String leafQueueName = application.getCSLeafQueue().getQueueName(); - getPreemptionManager().addKillableContainer( - new KillableContainer(killableContainer, node.getPartition(), + getPreemptionManager().removeKillableContainer( + new KillableContainer(nonKillableContainer, node.getPartition(), leafQueueName)); - } } - } - - private synchronized void markContainerForNonKillable( - RMContainer nonKillableContainer) { - if (LOG.isDebugEnabled()) { - LOG.debug( - SchedulerEventType.MARK_CONTAINER_FOR_NONKILLABLE + ": container" - + nonKillableContainer.toString()); - } - - FiCaSchedulerNode node = (FiCaSchedulerNode) getSchedulerNode( - nonKillableContainer.getAllocatedNode()); - - FiCaSchedulerApp application = getCurrentAttemptForContainer( - nonKillableContainer.getContainerId()); - - node.markContainerToNonKillable(nonKillableContainer.getContainerId()); - - // notify PreemptionManager - // Get the application for the finished container - if (null != application) { - String leafQueueName = application.getCSLeafQueue().getQueueName(); - getPreemptionManager().removeKillableContainer( - new KillableContainer(nonKillableContainer, node.getPartition(), - leafQueueName)); + } + } finally { + writeLock.unlock(); } } @Override - public synchronized boolean checkAccess(UserGroupInformation callerUGI, + public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, String queueName) { CSQueue queue = getQueue(queueName); if (queue == null) { if (LOG.isDebugEnabled()) { - LOG.debug("ACL not found for queue access-type " + acl - + " for queue " + queueName); + LOG.debug("ACL not found for queue access-type " + acl + " for queue " + + queueName); } return false; } @@ -1802,181 +1895,211 @@ public class CapacityScheduler extends return planQueueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX; } - private synchronized String resolveReservationQueueName(String queueName, + private String resolveReservationQueueName(String queueName, ApplicationId applicationId, ReservationId reservationID, boolean isRecovering) { - CSQueue queue = getQueue(queueName); - // Check if the queue is a plan queue - if ((queue == null) || !(queue instanceof PlanQueue)) { - return queueName; - } - if (reservationID != null) { - String resQName = reservationID.toString(); - queue = getQueue(resQName); - if (queue == null) { - // reservation has terminated during failover - if (isRecovering - && conf.getMoveOnExpiry(getQueue(queueName).getQueuePath())) { - // move to the default child queue of the plan - return getDefaultReservationQueueName(queueName); - } - String message = - "Application " + applicationId - + " submitted to a reservation which is not currently active: " - + resQName; - this.rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, - RMAppEventType.APP_REJECTED, message)); - return null; - } - if (!queue.getParent().getQueueName().equals(queueName)) { - String message = - "Application: " + applicationId + " submitted to a reservation " - + resQName + " which does not belong to the specified queue: " - + queueName; - this.rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, - RMAppEventType.APP_REJECTED, message)); - return null; - } - // use the reservation queue to run the app - queueName = resQName; - } else { - // use the default child queue of the plan for unreserved apps - queueName = getDefaultReservationQueueName(queueName); - } - return queueName; - } - - @Override - public synchronized void removeQueue(String queueName) - throws SchedulerDynamicEditException { - LOG.info("Removing queue: " + queueName); - CSQueue q = this.getQueue(queueName); - if (!(q instanceof ReservationQueue)) { - throw new SchedulerDynamicEditException("The queue that we are asked " - + "to remove (" + queueName + ") is not a ReservationQueue"); - } - ReservationQueue disposableLeafQueue = (ReservationQueue) q; - // at this point we should have no more apps - if (disposableLeafQueue.getNumApplications() > 0) { - throw new SchedulerDynamicEditException("The queue " + queueName - + " is not empty " + disposableLeafQueue.getApplications().size() - + " active apps " + disposableLeafQueue.getPendingApplications().size() - + " pending apps"); - } - - ((PlanQueue) disposableLeafQueue.getParent()).removeChildQueue(q); - this.queues.remove(queueName); - LOG.info("Removal of ReservationQueue " + queueName + " has succeeded"); - } - - @Override - public synchronized void addQueue(Queue queue) - throws SchedulerDynamicEditException { - - if (!(queue instanceof ReservationQueue)) { - throw new SchedulerDynamicEditException("Queue " + queue.getQueueName() - + " is not a ReservationQueue"); - } - - ReservationQueue newQueue = (ReservationQueue) queue; - - if (newQueue.getParent() == null - || !(newQueue.getParent() instanceof PlanQueue)) { - throw new SchedulerDynamicEditException("ParentQueue for " - + newQueue.getQueueName() - + " is not properly set (should be set and be a PlanQueue)"); - } - - PlanQueue parentPlan = (PlanQueue) newQueue.getParent(); - String queuename = newQueue.getQueueName(); - parentPlan.addChildQueue(newQueue); - this.queues.put(queuename, newQueue); - LOG.info("Creation of ReservationQueue " + newQueue + " succeeded"); - } - - @Override - public synchronized void setEntitlement(String inQueue, - QueueEntitlement entitlement) throws SchedulerDynamicEditException, - YarnException { - LeafQueue queue = getAndCheckLeafQueue(inQueue); - ParentQueue parent = (ParentQueue) queue.getParent(); - - if (!(queue instanceof ReservationQueue)) { - throw new SchedulerDynamicEditException("Entitlement can not be" - + " modified dynamically since queue " + inQueue - + " is not a ReservationQueue"); - } - - if (!(parent instanceof PlanQueue)) { - throw new SchedulerDynamicEditException("The parent of ReservationQueue " - + inQueue + " must be an PlanQueue"); - } - - ReservationQueue newQueue = (ReservationQueue) queue; - - float sumChilds = ((PlanQueue) parent).sumOfChildCapacities(); - float newChildCap = sumChilds - queue.getCapacity() + entitlement.getCapacity(); - - if (newChildCap >= 0 && newChildCap < 1.0f + CSQueueUtils.EPSILON) { - // note: epsilon checks here are not ok, as the epsilons might accumulate - // and become a problem in aggregate - if (Math.abs(entitlement.getCapacity() - queue.getCapacity()) == 0 - && Math.abs(entitlement.getMaxCapacity() - queue.getMaximumCapacity()) == 0) { - return; - } - newQueue.setEntitlement(entitlement); - } else { - throw new SchedulerDynamicEditException( - "Sum of child queues would exceed 100% for PlanQueue: " - + parent.getQueueName()); - } - LOG.info("Set entitlement for ReservationQueue " + inQueue + " to " - + queue.getCapacity() + " request was (" + entitlement.getCapacity() + ")"); - } - - @Override - public synchronized String moveApplication(ApplicationId appId, - String targetQueueName) throws YarnException { - FiCaSchedulerApp app = - getApplicationAttempt(ApplicationAttemptId.newInstance(appId, 0)); - String sourceQueueName = app.getQueue().getQueueName(); - LeafQueue source = getAndCheckLeafQueue(sourceQueueName); - String destQueueName = handleMoveToPlanQueue(targetQueueName); - LeafQueue dest = getAndCheckLeafQueue(destQueueName); - // Validation check - ACLs, submission limits for user & queue - String user = app.getUser(); - checkQueuePartition(app, dest); try { - dest.submitApplication(appId, user, destQueueName); - } catch (AccessControlException e) { - throw new YarnException(e); + readLock.lock(); + CSQueue queue = getQueue(queueName); + // Check if the queue is a plan queue + if ((queue == null) || !(queue instanceof PlanQueue)) { + return queueName; + } + if (reservationID != null) { + String resQName = reservationID.toString(); + queue = getQueue(resQName); + if (queue == null) { + // reservation has terminated during failover + if (isRecovering && conf.getMoveOnExpiry( + getQueue(queueName).getQueuePath())) { + // move to the default child queue of the plan + return getDefaultReservationQueueName(queueName); + } + String message = "Application " + applicationId + + " submitted to a reservation which is not currently active: " + + resQName; + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED, + message)); + return null; + } + if (!queue.getParent().getQueueName().equals(queueName)) { + String message = + "Application: " + applicationId + " submitted to a reservation " + + resQName + " which does not belong to the specified queue: " + + queueName; + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED, + message)); + return null; + } + // use the reservation queue to run the app + queueName = resQName; + } else{ + // use the default child queue of the plan for unreserved apps + queueName = getDefaultReservationQueueName(queueName); + } + return queueName; + } finally { + readLock.unlock(); } - // Move all live containers - for (RMContainer rmContainer : app.getLiveContainers()) { - source.detachContainer(getClusterResource(), app, rmContainer); - // attach the Container to another queue - dest.attachContainer(getClusterResource(), app, rmContainer); + + } + + @Override + public void removeQueue(String queueName) + throws SchedulerDynamicEditException { + try { + writeLock.lock(); + LOG.info("Removing queue: " + queueName); + CSQueue q = this.getQueue(queueName); + if (!(q instanceof ReservationQueue)) { + throw new SchedulerDynamicEditException( + "The queue that we are asked " + "to remove (" + queueName + + ") is not a ReservationQueue"); + } + ReservationQueue disposableLeafQueue = (ReservationQueue) q; + // at this point we should have no more apps + if (disposableLeafQueue.getNumApplications() > 0) { + throw new SchedulerDynamicEditException( + "The queue " + queueName + " is not empty " + disposableLeafQueue + .getApplications().size() + " active apps " + + disposableLeafQueue.getPendingApplications().size() + + " pending apps"); + } + + ((PlanQueue) disposableLeafQueue.getParent()).removeChildQueue(q); + this.queues.remove(queueName); + LOG.info("Removal of ReservationQueue " + queueName + " has succeeded"); + } finally { + writeLock.unlock(); + } + } + + @Override + public void addQueue(Queue queue) + throws SchedulerDynamicEditException { + try { + writeLock.lock(); + if (!(queue instanceof ReservationQueue)) { + throw new SchedulerDynamicEditException( + "Queue " + queue.getQueueName() + " is not a ReservationQueue"); + } + + ReservationQueue newQueue = (ReservationQueue) queue; + + if (newQueue.getParent() == null || !(newQueue + .getParent() instanceof PlanQueue)) { + throw new SchedulerDynamicEditException( + "ParentQueue for " + newQueue.getQueueName() + + " is not properly set (should be set and be a PlanQueue)"); + } + + PlanQueue parentPlan = (PlanQueue) newQueue.getParent(); + String queuename = newQueue.getQueueName(); + parentPlan.addChildQueue(newQueue); + this.queues.put(queuename, newQueue); + LOG.info("Creation of ReservationQueue " + newQueue + " succeeded"); + } finally { + writeLock.unlock(); + } + } + + @Override + public void setEntitlement(String inQueue, QueueEntitlement entitlement) + throws YarnException { + try { + writeLock.lock(); + LeafQueue queue = getAndCheckLeafQueue(inQueue); + ParentQueue parent = (ParentQueue) queue.getParent(); + + if (!(queue instanceof ReservationQueue)) { + throw new SchedulerDynamicEditException( + "Entitlement can not be" + " modified dynamically since queue " + + inQueue + " is not a ReservationQueue"); + } + + if (!(parent instanceof PlanQueue)) { + throw new SchedulerDynamicEditException( + "The parent of ReservationQueue " + inQueue + + " must be an PlanQueue"); + } + + ReservationQueue newQueue = (ReservationQueue) queue; + + float sumChilds = ((PlanQueue) parent).sumOfChildCapacities(); + float newChildCap = + sumChilds - queue.getCapacity() + entitlement.getCapacity(); + + if (newChildCap >= 0 && newChildCap < 1.0f + CSQueueUtils.EPSILON) { + // note: epsilon checks here are not ok, as the epsilons might + // accumulate and become a problem in aggregate + if (Math.abs(entitlement.getCapacity() - queue.getCapacity()) == 0 + && Math.abs( + entitlement.getMaxCapacity() - queue.getMaximumCapacity()) == 0) { + return; + } + newQueue.setEntitlement(entitlement); + } else{ + throw new SchedulerDynamicEditException( + "Sum of child queues would exceed 100% for PlanQueue: " + parent + .getQueueName()); + } + LOG.info( + "Set entitlement for ReservationQueue " + inQueue + " to " + queue + .getCapacity() + " request was (" + entitlement.getCapacity() + + ")"); + } finally { + writeLock.unlock(); + } + } + + @Override + public String moveApplication(ApplicationId appId, + String targetQueueName) throws YarnException { + try { + writeLock.lock(); + FiCaSchedulerApp app = getApplicationAttempt( + ApplicationAttemptId.newInstance(appId, 0)); + String sourceQueueName = app.getQueue().getQueueName(); + LeafQueue source = getAndCheckLeafQueue(sourceQueueName); + String destQueueName = handleMoveToPlanQueue(targetQueueName); + LeafQueue dest = getAndCheckLeafQueue(destQueueName); + // Validation check - ACLs, submission limits for user & queue + String user = app.getUser(); + checkQueuePartition(app, dest); + try { + dest.submitApplication(appId, user, destQueueName); + } catch (AccessControlException e) { + throw new YarnException(e); + } + // Move all live containers + for (RMContainer rmContainer : app.getLiveContainers()) { + source.detachContainer(getClusterResource(), app, rmContainer); + // attach the Container to another queue + dest.attachContainer(getClusterResource(), app, rmContainer); + } + // Detach the application.. + source.finishApplicationAttempt(app, sourceQueueName); + source.getParent().finishApplication(appId, app.getUser()); + // Finish app & update metrics + app.move(dest); + // Submit to a new queue + dest.submitApplicationAttempt(app, user); + applications.get(appId).setQueue(dest); + LOG.info("App: " + app.getApplicationId() + " successfully moved from " + + sourceQueueName + " to: " + destQueueName); + return targetQueueName; + } finally { + writeLock.unlock(); } - // Detach the application.. - source.finishApplicationAttempt(app, sourceQueueName); - source.getParent().finishApplication(appId, app.getUser()); - // Finish app & update metrics - app.move(dest); - // Submit to a new queue - dest.submitApplicationAttempt(app, user); - applications.get(appId).setQueue(dest); - LOG.info("App: " + app.getApplicationId() + " successfully moved from " - + sourceQueueName + " to: " + destQueueName); - return targetQueueName; } /** * Check application can be moved to queue with labels enabled. All labels in * application life time will be checked * - * @param appId + * @param app * @param dest * @throws YarnException */ @@ -2166,16 +2289,8 @@ public class CapacityScheduler extends // As we use iterator over a TreeSet for OrderingPolicy, once we change // priority then reinsert back to make order correct. LeafQueue queue = (LeafQueue) getQueue(rmApp.getQueue()); - synchronized (queue) { - queue.getOrderingPolicy().removeSchedulableEntity( - application.getCurrentAppAttempt()); - // Update new priority in SchedulerApplication - application.setPriority(appPriority); - - queue.getOrderingPolicy().addSchedulableEntity( - application.getCurrentAppAttempt()); - } + queue.updateApplicationPriority(application, appPriority); // Update the changed application state to timeline server rmContext.getSystemMetricsPublisher().appUpdated(rmApp, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 6129772a5b1..eecd4ba4a3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -2227,6 +2227,22 @@ public class LeafQueue extends AbstractCSQueue { } } + public void updateApplicationPriority(SchedulerApplication app, + Priority newAppPriority) { + try { + writeLock.lock(); + FiCaSchedulerApp attempt = app.getCurrentAppAttempt(); + getOrderingPolicy().removeSchedulableEntity(attempt); + + // Update new priority in SchedulerApplication + attempt.setPriority(newAppPriority); + + getOrderingPolicy().addSchedulableEntity(attempt); + } finally { + writeLock.unlock(); + } + } + public OrderingPolicy getPendingAppsOrderingPolicy() { return pendingOrderingPolicy; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index fd43e748fc2..aa7ad500a49 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -23,6 +23,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -666,6 +667,9 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { } finally { writeLock.unlock(); } + } + public ReentrantReadWriteLock.WriteLock getWriteLock() { + return this.writeLock; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java index 017a256e043..b80a17cdf76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -37,6 +38,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * The NodeQueueLoadMonitor keeps track of load metrics (such as queue length @@ -103,16 +105,23 @@ public class NodeQueueLoadMonitor implements ClusterMonitor { new ConcurrentHashMap<>(); private final LoadComparator comparator; private QueueLimitCalculator thresholdCalculator; + private ReentrantReadWriteLock sortedNodesLock = new ReentrantReadWriteLock(); + private ReentrantReadWriteLock clusterNodesLock = + new ReentrantReadWriteLock(); Runnable computeTask = new Runnable() { @Override public void run() { - synchronized (sortedNodes) { + ReentrantReadWriteLock.WriteLock writeLock = sortedNodesLock.writeLock(); + writeLock.lock(); + try { sortedNodes.clear(); sortedNodes.addAll(sortNodes()); if (thresholdCalculator != null) { thresholdCalculator.update(); } + } finally { + writeLock.unlock(); } } }; @@ -166,9 +175,16 @@ public class NodeQueueLoadMonitor implements ClusterMonitor { @Override public void removeNode(RMNode removedRMNode) { LOG.debug("Node delete event for: " + removedRMNode.getNode().getName()); - synchronized (this.clusterNodes) { - if (this.clusterNodes.containsKey(removedRMNode.getNodeID())) { - this.clusterNodes.remove(removedRMNode.getNodeID()); + ReentrantReadWriteLock.WriteLock writeLock = clusterNodesLock.writeLock(); + writeLock.lock(); + ClusterNode node; + try { + node = this.clusterNodes.remove(removedRMNode.getNodeID()); + } finally { + writeLock.unlock(); + } + if (LOG.isDebugEnabled()) { + if (node != null) { LOG.debug("Delete ClusterNode: " + removedRMNode.getNodeID()); } else { LOG.debug("Node not in list!"); @@ -186,7 +202,9 @@ public class NodeQueueLoadMonitor implements ClusterMonitor { int waitQueueLength = queuedContainersStatus.getWaitQueueLength(); // Add nodes to clusterNodes. If estimatedQueueTime is -1, ignore node // UNLESS comparator is based on queue length. - synchronized (this.clusterNodes) { + ReentrantReadWriteLock.WriteLock writeLock = clusterNodesLock.writeLock(); + writeLock.lock(); + try { ClusterNode currentNode = this.clusterNodes.get(rmNode.getNodeID()); if (currentNode == null) { if (estimatedQueueWaitTime != -1 @@ -222,6 +240,8 @@ public class NodeQueueLoadMonitor implements ClusterMonitor { "wait queue length [" + currentNode.queueLength + "]"); } } + } finally { + writeLock.unlock(); } } @@ -245,15 +265,22 @@ public class NodeQueueLoadMonitor implements ClusterMonitor { * @return ordered list of nodes */ public List selectLeastLoadedNodes(int k) { - synchronized (this.sortedNodes) { - return ((k < this.sortedNodes.size()) && (k >= 0)) ? + ReentrantReadWriteLock.ReadLock readLock = sortedNodesLock.readLock(); + readLock.lock(); + try { + List retVal = ((k < this.sortedNodes.size()) && (k >= 0)) ? new ArrayList<>(this.sortedNodes).subList(0, k) : new ArrayList<>(this.sortedNodes); + return Collections.unmodifiableList(retVal); + } finally { + readLock.unlock(); } } private List sortNodes() { - synchronized (this.clusterNodes) { + ReentrantReadWriteLock.ReadLock readLock = clusterNodesLock.readLock(); + readLock.lock(); + try { ArrayList aList = new ArrayList<>(this.clusterNodes.values()); List retList = new ArrayList<>(); Object[] nodes = aList.toArray(); @@ -267,6 +294,8 @@ public class NodeQueueLoadMonitor implements ClusterMonitor { retList.add(((ClusterNode)nodes[j]).nodeId); } return retList; + } finally { + readLock.unlock(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index a6adb47139c..9d5bbe50f25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -295,7 +295,7 @@ public class FSLeafQueue extends FSQueue { Resource toAdd = sched.getDemand(); if (LOG.isDebugEnabled()) { LOG.debug("Counting resource from " + sched.getName() + " " + toAdd - + "; Total resource consumption for " + getName() + " now " + + "; Total resource demand for " + getName() + " now " + demand); } demand = Resources.add(demand, toAdd); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index e58c3f1a686..d05390b7cae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -158,13 +158,13 @@ public class FSParentQueue extends FSQueue { for (FSQueue childQueue : childQueues) { childQueue.updateDemand(); Resource toAdd = childQueue.getDemand(); - if (LOG.isDebugEnabled()) { - LOG.debug("Counting resource from " + childQueue.getName() + " " + - toAdd + "; Total resource consumption for " + getName() + - " now " + demand); - } demand = Resources.add(demand, toAdd); demand = Resources.componentwiseMin(demand, maxShare); + if (LOG.isDebugEnabled()) { + LOG.debug("Counting resource from " + childQueue.getName() + " " + + toAdd + "; Total resource demand for " + getName() + + " now " + demand); + } if (Resources.equals(demand, maxShare)) { break; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 310f2f92e05..8daf0f333c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -186,10 +186,13 @@ public class FairScheduler extends // an app can be reserved on protected boolean sizeBasedWeight; // Give larger weights to larger jobs - protected boolean continuousSchedulingEnabled; // Continuous Scheduling enabled or not - protected int continuousSchedulingSleepMs; // Sleep time for each pass in continuous scheduling + // Continuous Scheduling enabled or not + protected boolean continuousSchedulingEnabled; + // Sleep time for each pass in continuous scheduling + protected volatile int continuousSchedulingSleepMs; + // Node available resource comparator private Comparator nodeAvailableResourceComparator = - new NodeAvailableResourceComparator(); // Node available resource comparator + new NodeAvailableResourceComparator(); protected double nodeLocalityThreshold; // Cluster threshold for node locality protected double rackLocalityThreshold; // Cluster threshold for rack locality protected long nodeLocalityDelayMs; // Delay for node locality @@ -338,36 +341,40 @@ public class FairScheduler extends * fair shares, deficits, minimum slot allocations, and amount of used and * required resources per job. */ - protected synchronized void update() { - long start = getClock().getTime(); - updateStarvationStats(); // Determine if any queues merit preemption + protected void update() { + try { + writeLock.lock(); + long start = getClock().getTime(); + updateStarvationStats(); // Determine if any queues merit preemption - FSQueue rootQueue = queueMgr.getRootQueue(); + FSQueue rootQueue = queueMgr.getRootQueue(); - // Recursively update demands for all queues - rootQueue.updateDemand(); + // Recursively update demands for all queues + rootQueue.updateDemand(); - Resource clusterResource = getClusterResource(); - rootQueue.setFairShare(clusterResource); - // Recursively compute fair shares for all queues - // and update metrics - rootQueue.recomputeShares(); - updateRootQueueMetrics(); + Resource clusterResource = getClusterResource(); + rootQueue.setFairShare(clusterResource); + // Recursively compute fair shares for all queues + // and update metrics + rootQueue.recomputeShares(); + updateRootQueueMetrics(); - if (LOG.isDebugEnabled()) { - if (--updatesToSkipForDebug < 0) { - updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY; - LOG.debug("Cluster Capacity: " + clusterResource + - " Allocations: " + rootMetrics.getAllocatedResources() + - " Availability: " + Resource.newInstance( - rootMetrics.getAvailableMB(), - rootMetrics.getAvailableVirtualCores()) + - " Demand: " + rootQueue.getDemand()); + if (LOG.isDebugEnabled()) { + if (--updatesToSkipForDebug < 0) { + updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY; + LOG.debug("Cluster Capacity: " + clusterResource + " Allocations: " + + rootMetrics.getAllocatedResources() + " Availability: " + + Resource.newInstance(rootMetrics.getAvailableMB(), + rootMetrics.getAvailableVirtualCores()) + " Demand: " + rootQueue + .getDemand()); + } } - } - long duration = getClock().getTime() - start; - fsOpDurations.addUpdateCallDuration(duration); + long duration = getClock().getTime() - start; + fsOpDurations.addUpdateCallDuration(duration); + } finally { + writeLock.unlock(); + } } /** @@ -389,23 +396,28 @@ public class FairScheduler extends * such queues exist, compute how many tasks of each type need to be preempted * and then select the right ones using preemptTasks. */ - protected synchronized void preemptTasksIfNecessary() { - if (!shouldAttemptPreemption()) { - return; - } + protected void preemptTasksIfNecessary() { + try { + writeLock.lock(); + if (!shouldAttemptPreemption()) { + return; + } - long curTime = getClock().getTime(); - if (curTime - lastPreemptCheckTime < preemptionInterval) { - return; - } - lastPreemptCheckTime = curTime; + long curTime = getClock().getTime(); + if (curTime - lastPreemptCheckTime < preemptionInterval) { + return; + } + lastPreemptCheckTime = curTime; - Resource resToPreempt = Resources.clone(Resources.none()); - for (FSLeafQueue sched : queueMgr.getLeafQueues()) { - Resources.addTo(resToPreempt, resourceDeficit(sched, curTime)); - } - if (isResourceGreaterThanNone(resToPreempt)) { - preemptResources(resToPreempt); + Resource resToPreempt = Resources.clone(Resources.none()); + for (FSLeafQueue sched : queueMgr.getLeafQueues()) { + Resources.addTo(resToPreempt, resourceDeficit(sched, curTime)); + } + if (isResourceGreaterThanNone(resToPreempt)) { + preemptResources(resToPreempt); + } + } finally { + writeLock.unlock(); } } @@ -549,22 +561,27 @@ public class FairScheduler extends return deficit; } - public synchronized RMContainerTokenSecretManager + public RMContainerTokenSecretManager getContainerTokenSecretManager() { return rmContext.getContainerTokenSecretManager(); } - // synchronized for sizeBasedWeight - public synchronized ResourceWeights getAppWeight(FSAppAttempt app) { - double weight = 1.0; - if (sizeBasedWeight) { - // Set weight based on current memory demand - weight = Math.log1p(app.getDemand().getMemorySize()) / Math.log(2); + public ResourceWeights getAppWeight(FSAppAttempt app) { + try { + readLock.lock(); + double weight = 1.0; + if (sizeBasedWeight) { + // Set weight based on current memory demand + weight = Math.log1p(app.getDemand().getMemorySize()) / Math.log(2); + } + weight *= app.getPriority().getPriority(); + ResourceWeights resourceWeights = app.getResourceWeights(); + resourceWeights.setWeight((float) weight); + return resourceWeights; + } finally { + readLock.unlock(); } - weight *= app.getPriority().getPriority(); - ResourceWeights resourceWeights = app.getResourceWeights(); - resourceWeights.setWeight((float)weight); - return resourceWeights; + } public Resource getIncrementResourceCapability() { @@ -595,7 +612,7 @@ public class FairScheduler extends return continuousSchedulingEnabled; } - public synchronized int getContinuousSchedulingSleepMs() { + public int getContinuousSchedulingSleepMs() { return continuousSchedulingSleepMs; } @@ -617,113 +634,123 @@ public class FairScheduler extends * user. This will accept a new app even if the user or queue is above * configured limits, but the app will not be marked as runnable. */ - protected synchronized void addApplication(ApplicationId applicationId, + protected void addApplication(ApplicationId applicationId, String queueName, String user, boolean isAppRecovering) { if (queueName == null || queueName.isEmpty()) { - String message = "Reject application " + applicationId + - " submitted by user " + user + " with an empty queue name."; + String message = + "Reject application " + applicationId + " submitted by user " + user + + " with an empty queue name."; LOG.info(message); - rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, - RMAppEventType.APP_REJECTED, message)); + rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED, + message)); return; } if (queueName.startsWith(".") || queueName.endsWith(".")) { - String message = "Reject application " + applicationId - + " submitted by user " + user + " with an illegal queue name " - + queueName + ". " - + "The queue name cannot start/end with period."; + String message = + "Reject application " + applicationId + " submitted by user " + user + + " with an illegal queue name " + queueName + ". " + + "The queue name cannot start/end with period."; LOG.info(message); - rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, - RMAppEventType.APP_REJECTED, message)); + rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED, + message)); return; } - RMApp rmApp = rmContext.getRMApps().get(applicationId); - FSLeafQueue queue = assignToQueue(rmApp, queueName, user); - if (queue == null) { - return; - } - - // Enforce ACLs - UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user); - - if (!queue.hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi) - && !queue.hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) { - String msg = "User " + userUgi.getUserName() + - " cannot submit applications to queue " + queue.getName() + - "(requested queuename is " + queueName + ")"; - LOG.info(msg); - rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, - RMAppEventType.APP_REJECTED, msg)); - return; - } - - SchedulerApplication application = - new SchedulerApplication(queue, user); - applications.put(applicationId, application); - queue.getMetrics().submitApp(user); - - LOG.info("Accepted application " + applicationId + " from user: " + user - + ", in queue: " + queueName + ", currently num of applications: " - + applications.size()); - if (isAppRecovering) { - if (LOG.isDebugEnabled()) { - LOG.debug(applicationId + " is recovering. Skip notifying APP_ACCEPTED"); + try { + writeLock.lock(); + RMApp rmApp = rmContext.getRMApps().get(applicationId); + FSLeafQueue queue = assignToQueue(rmApp, queueName, user); + if (queue == null) { + return; } - } else { - rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED)); + + // Enforce ACLs + UserGroupInformation userUgi = UserGroupInformation.createRemoteUser( + user); + + if (!queue.hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi) && !queue + .hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) { + String msg = "User " + userUgi.getUserName() + + " cannot submit applications to queue " + queue.getName() + + "(requested queuename is " + queueName + ")"; + LOG.info(msg); + rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED, msg)); + return; + } + + SchedulerApplication application = + new SchedulerApplication(queue, user); + applications.put(applicationId, application); + queue.getMetrics().submitApp(user); + + LOG.info("Accepted application " + applicationId + " from user: " + user + + ", in queue: " + queue.getName() + + ", currently num of applications: " + applications.size()); + if (isAppRecovering) { + if (LOG.isDebugEnabled()) { + LOG.debug(applicationId + + " is recovering. Skip notifying APP_ACCEPTED"); + } + } else{ + rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED)); + } + } finally { + writeLock.unlock(); } } /** * Add a new application attempt to the scheduler. */ - protected synchronized void addApplicationAttempt( + protected void addApplicationAttempt( ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt, boolean isAttemptRecovering) { - SchedulerApplication application = - applications.get(applicationAttemptId.getApplicationId()); - String user = application.getUser(); - FSLeafQueue queue = (FSLeafQueue) application.getQueue(); + try { + writeLock.lock(); + SchedulerApplication application = applications.get( + applicationAttemptId.getApplicationId()); + String user = application.getUser(); + FSLeafQueue queue = (FSLeafQueue) application.getQueue(); - FSAppAttempt attempt = - new FSAppAttempt(this, applicationAttemptId, user, - queue, new ActiveUsersManager(getRootQueueMetrics()), - rmContext); - if (transferStateFromPreviousAttempt) { - attempt.transferStateFromPreviousAttempt(application - .getCurrentAppAttempt()); - } - application.setCurrentAppAttempt(attempt); - - boolean runnable = maxRunningEnforcer.canAppBeRunnable(queue, user); - queue.addApp(attempt, runnable); - if (runnable) { - maxRunningEnforcer.trackRunnableApp(attempt); - } else { - maxRunningEnforcer.trackNonRunnableApp(attempt); - } - - queue.getMetrics().submitAppAttempt(user); - - LOG.info("Added Application Attempt " + applicationAttemptId - + " to scheduler from user: " + user); - - if (isAttemptRecovering) { - if (LOG.isDebugEnabled()) { - LOG.debug(applicationAttemptId - + " is recovering. Skipping notifying ATTEMPT_ADDED"); + FSAppAttempt attempt = new FSAppAttempt(this, applicationAttemptId, user, + queue, new ActiveUsersManager(getRootQueueMetrics()), rmContext); + if (transferStateFromPreviousAttempt) { + attempt.transferStateFromPreviousAttempt( + application.getCurrentAppAttempt()); } - } else { - rmContext.getDispatcher().getEventHandler().handle( - new RMAppAttemptEvent(applicationAttemptId, - RMAppAttemptEventType.ATTEMPT_ADDED)); + application.setCurrentAppAttempt(attempt); + + boolean runnable = maxRunningEnforcer.canAppBeRunnable(queue, user); + queue.addApp(attempt, runnable); + if (runnable) { + maxRunningEnforcer.trackRunnableApp(attempt); + } else{ + maxRunningEnforcer.trackNonRunnableApp(attempt); + } + + queue.getMetrics().submitAppAttempt(user); + + LOG.info("Added Application Attempt " + applicationAttemptId + + " to scheduler from user: " + user); + + if (isAttemptRecovering) { + if (LOG.isDebugEnabled()) { + LOG.debug(applicationAttemptId + + " is recovering. Skipping notifying ATTEMPT_ADDED"); + } + } else{ + rmContext.getDispatcher().getEventHandler().handle( + new RMAppAttemptEvent(applicationAttemptId, + RMAppAttemptEventType.ATTEMPT_ADDED)); + } + } finally { + writeLock.unlock(); } } @@ -769,70 +796,71 @@ public class FairScheduler extends return queue; } - private synchronized void removeApplication(ApplicationId applicationId, + private void removeApplication(ApplicationId applicationId, RMAppState finalState) { - SchedulerApplication application = - applications.get(applicationId); - if (application == null){ + SchedulerApplication application = applications.remove( + applicationId); + if (application == null) { LOG.warn("Couldn't find application " + applicationId); - return; + } else{ + application.stop(finalState); } - application.stop(finalState); - applications.remove(applicationId); } - private synchronized void removeApplicationAttempt( + private void removeApplicationAttempt( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) { - LOG.info("Application " + applicationAttemptId + " is done." + - " finalState=" + rmAppAttemptFinalState); - SchedulerApplication application = - applications.get(applicationAttemptId.getApplicationId()); - FSAppAttempt attempt = getSchedulerApp(applicationAttemptId); + try { + writeLock.lock(); + LOG.info( + "Application " + applicationAttemptId + " is done." + " finalState=" + + rmAppAttemptFinalState); + FSAppAttempt attempt = getApplicationAttempt(applicationAttemptId); - if (attempt == null || application == null) { - LOG.info("Unknown application " + applicationAttemptId + " has completed!"); - return; - } - - // Release all the running containers - for (RMContainer rmContainer : attempt.getLiveContainers()) { - if (keepContainers - && rmContainer.getState().equals(RMContainerState.RUNNING)) { - // do not kill the running container in the case of work-preserving AM - // restart. - LOG.info("Skip killing " + rmContainer.getContainerId()); - continue; + if (attempt == null) { + LOG.info( + "Unknown application " + applicationAttemptId + " has completed!"); + return; } - super.completedContainer(rmContainer, - SchedulerUtils.createAbnormalContainerStatus( - rmContainer.getContainerId(), - SchedulerUtils.COMPLETED_APPLICATION), - RMContainerEventType.KILL); - } - // Release all reserved containers - for (RMContainer rmContainer : attempt.getReservedContainers()) { - super.completedContainer(rmContainer, - SchedulerUtils.createAbnormalContainerStatus( - rmContainer.getContainerId(), - "Application Complete"), - RMContainerEventType.KILL); - } - // Clean up pending requests, metrics etc. - attempt.stop(rmAppAttemptFinalState); + // Release all the running containers + for (RMContainer rmContainer : attempt.getLiveContainers()) { + if (keepContainers && rmContainer.getState().equals( + RMContainerState.RUNNING)) { + // do not kill the running container in the case of work-preserving AM + // restart. + LOG.info("Skip killing " + rmContainer.getContainerId()); + continue; + } + super.completedContainer(rmContainer, SchedulerUtils + .createAbnormalContainerStatus(rmContainer.getContainerId(), + SchedulerUtils.COMPLETED_APPLICATION), + RMContainerEventType.KILL); + } - // Inform the queue - FSLeafQueue queue = queueMgr.getLeafQueue(attempt.getQueue() - .getQueueName(), false); - boolean wasRunnable = queue.removeApp(attempt); + // Release all reserved containers + for (RMContainer rmContainer : attempt.getReservedContainers()) { + super.completedContainer(rmContainer, SchedulerUtils + .createAbnormalContainerStatus(rmContainer.getContainerId(), + "Application Complete"), RMContainerEventType.KILL); + } + // Clean up pending requests, metrics etc. + attempt.stop(rmAppAttemptFinalState); - if (wasRunnable) { - maxRunningEnforcer.untrackRunnableApp(attempt); - maxRunningEnforcer.updateRunnabilityOnAppRemoval(attempt, - attempt.getQueue()); - } else { - maxRunningEnforcer.untrackNonRunnableApp(attempt); + // Inform the queue + FSLeafQueue queue = queueMgr.getLeafQueue( + attempt.getQueue().getQueueName(), false); + boolean wasRunnable = queue.removeApp(attempt); + + if (wasRunnable) { + maxRunningEnforcer.untrackRunnableApp(attempt); + maxRunningEnforcer.updateRunnabilityOnAppRemoval(attempt, + attempt.getQueue()); + } else{ + maxRunningEnforcer.untrackNonRunnableApp(attempt); + } + } finally { + writeLock.unlock(); } } @@ -840,97 +868,108 @@ public class FairScheduler extends * Clean up a completed container. */ @Override - protected synchronized void completedContainerInternal( + protected void completedContainerInternal( RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) { + try { + writeLock.lock(); + Container container = rmContainer.getContainer(); - Container container = rmContainer.getContainer(); + // Get the application for the finished container + FSAppAttempt application = getCurrentAttemptForContainer( + container.getId()); + ApplicationId appId = + container.getId().getApplicationAttemptId().getApplicationId(); + if (application == null) { + LOG.info( + "Container " + container + " of" + " finished application " + appId + + " completed with event " + event); + return; + } - // Get the application for the finished container - FSAppAttempt application = - getCurrentAttemptForContainer(container.getId()); - ApplicationId appId = - container.getId().getApplicationAttemptId().getApplicationId(); - if (application == null) { - LOG.info("Container " + container + " of" + - " finished application " + appId + - " completed with event " + event); - return; - } + // Get the node on which the container was allocated + FSSchedulerNode node = getFSSchedulerNode(container.getNodeId()); - // Get the node on which the container was allocated - FSSchedulerNode node = getFSSchedulerNode(container.getNodeId()); + if (rmContainer.getState() == RMContainerState.RESERVED) { + application.unreserve(rmContainer.getReservedSchedulerKey(), node); + } else{ + application.containerCompleted(rmContainer, containerStatus, event); + node.releaseContainer(container); + updateRootQueueMetrics(); + } - if (rmContainer.getState() == RMContainerState.RESERVED) { - application.unreserve(rmContainer.getReservedSchedulerKey(), node); - } else { - application.containerCompleted(rmContainer, containerStatus, event); - node.releaseContainer(container); - updateRootQueueMetrics(); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("Application attempt " + application.getApplicationAttemptId() - + " released container " + container.getId() + " on node: " + node - + " with event: " + event); + if (LOG.isDebugEnabled()) { + LOG.debug("Application attempt " + application.getApplicationAttemptId() + + " released container " + container.getId() + " on node: " + node + + " with event: " + event); + } + } finally { + writeLock.unlock(); } } - private synchronized void addNode(List containerReports, + private void addNode(List containerReports, RMNode node) { - FSSchedulerNode schedulerNode = new FSSchedulerNode(node, usePortForNodeName); - nodeTracker.addNode(schedulerNode); + try { + writeLock.lock(); + FSSchedulerNode schedulerNode = new FSSchedulerNode(node, + usePortForNodeName); + nodeTracker.addNode(schedulerNode); - triggerUpdate(); + triggerUpdate(); - Resource clusterResource = getClusterResource(); - queueMgr.getRootQueue().setSteadyFairShare(clusterResource); - queueMgr.getRootQueue().recomputeSteadyShares(); - LOG.info("Added node " + node.getNodeAddress() + - " cluster capacity: " + clusterResource); + Resource clusterResource = getClusterResource(); + queueMgr.getRootQueue().setSteadyFairShare(clusterResource); + queueMgr.getRootQueue().recomputeSteadyShares(); + LOG.info("Added node " + node.getNodeAddress() + " cluster capacity: " + + clusterResource); - recoverContainersOnNode(containerReports, node); - updateRootQueueMetrics(); + recoverContainersOnNode(containerReports, node); + updateRootQueueMetrics(); + } finally { + writeLock.unlock(); + } } - private synchronized void removeNode(RMNode rmNode) { - NodeId nodeId = rmNode.getNodeID(); - FSSchedulerNode node = nodeTracker.getNode(nodeId); - if (node == null) { - LOG.error("Attempting to remove non-existent node " + nodeId); - return; + private void removeNode(RMNode rmNode) { + try { + writeLock.lock(); + NodeId nodeId = rmNode.getNodeID(); + FSSchedulerNode node = nodeTracker.getNode(nodeId); + if (node == null) { + LOG.error("Attempting to remove non-existent node " + nodeId); + return; + } + + // Remove running containers + List runningContainers = + node.getCopiedListOfRunningContainers(); + for (RMContainer container : runningContainers) { + super.completedContainer(container, SchedulerUtils + .createAbnormalContainerStatus(container.getContainerId(), + SchedulerUtils.LOST_CONTAINER), RMContainerEventType.KILL); + } + + // Remove reservations, if any + RMContainer reservedContainer = node.getReservedContainer(); + if (reservedContainer != null) { + super.completedContainer(reservedContainer, SchedulerUtils + .createAbnormalContainerStatus(reservedContainer.getContainerId(), + SchedulerUtils.LOST_CONTAINER), RMContainerEventType.KILL); + } + + nodeTracker.removeNode(nodeId); + Resource clusterResource = getClusterResource(); + queueMgr.getRootQueue().setSteadyFairShare(clusterResource); + queueMgr.getRootQueue().recomputeSteadyShares(); + updateRootQueueMetrics(); + triggerUpdate(); + + LOG.info("Removed node " + rmNode.getNodeAddress() + " cluster capacity: " + + clusterResource); + } finally { + writeLock.unlock(); } - - // Remove running containers - List runningContainers = - node.getCopiedListOfRunningContainers(); - for (RMContainer container : runningContainers) { - super.completedContainer(container, - SchedulerUtils.createAbnormalContainerStatus( - container.getContainerId(), - SchedulerUtils.LOST_CONTAINER), - RMContainerEventType.KILL); - } - - // Remove reservations, if any - RMContainer reservedContainer = node.getReservedContainer(); - if (reservedContainer != null) { - super.completedContainer(reservedContainer, - SchedulerUtils.createAbnormalContainerStatus( - reservedContainer.getContainerId(), - SchedulerUtils.LOST_CONTAINER), - RMContainerEventType.KILL); - } - - nodeTracker.removeNode(nodeId); - Resource clusterResource = getClusterResource(); - queueMgr.getRootQueue().setSteadyFairShare(clusterResource); - queueMgr.getRootQueue().recomputeSteadyShares(); - updateRootQueueMetrics(); - triggerUpdate(); - - LOG.info("Removed node " + rmNode.getNodeAddress() + - " cluster capacity: " + clusterResource); } @Override @@ -959,12 +998,13 @@ public class FairScheduler extends // Release containers releaseContainers(release, application); - synchronized (application) { + try { + application.getWriteLock().lock(); if (!ask.isEmpty()) { if (LOG.isDebugEnabled()) { - LOG.debug("allocate: pre-update" + - " applicationAttemptId=" + appAttemptId + - " application=" + application.getApplicationId()); + LOG.debug( + "allocate: pre-update" + " applicationAttemptId=" + appAttemptId + + " application=" + application.getApplicationId()); } application.showRequests(); @@ -973,98 +1013,107 @@ public class FairScheduler extends application.showRequests(); } - - if (LOG.isDebugEnabled()) { - LOG.debug("allocate: post-update" + - " applicationAttemptId=" + appAttemptId + - " #ask=" + ask.size() + - " reservation= " + application.getCurrentReservation()); - - LOG.debug("Preempting " + application.getPreemptionContainers().size() - + " container(s)"); - } - - Set preemptionContainerIds = new HashSet(); - for (RMContainer container : application.getPreemptionContainers()) { - preemptionContainerIds.add(container.getContainerId()); - } - - application.updateBlacklist(blacklistAdditions, blacklistRemovals); - - List newlyAllocatedContainers = - application.pullNewlyAllocatedContainers(); - // Record container allocation time - if (!(newlyAllocatedContainers.isEmpty())) { - application.recordContainerAllocationTime(getClock().getTime()); - } - - Resource headroom = application.getHeadroom(); - application.setApplicationHeadroomForMetrics(headroom); - return new Allocation(newlyAllocatedContainers, headroom, - preemptionContainerIds, null, null, application.pullUpdatedNMTokens()); + } finally { + application.getWriteLock().unlock(); } + + if (LOG.isDebugEnabled()) { + LOG.debug( + "allocate: post-update" + " applicationAttemptId=" + appAttemptId + + " #ask=" + ask.size() + " reservation= " + application + .getCurrentReservation()); + + LOG.debug("Preempting " + application.getPreemptionContainers().size() + + " container(s)"); + } + + Set preemptionContainerIds = new HashSet(); + for (RMContainer container : application.getPreemptionContainers()) { + preemptionContainerIds.add(container.getContainerId()); + } + + application.updateBlacklist(blacklistAdditions, blacklistRemovals); + + List newlyAllocatedContainers = + application.pullNewlyAllocatedContainers(); + // Record container allocation time + if (!(newlyAllocatedContainers.isEmpty())) { + application.recordContainerAllocationTime(getClock().getTime()); + } + + Resource headroom = application.getHeadroom(); + application.setApplicationHeadroomForMetrics(headroom); + return new Allocation(newlyAllocatedContainers, headroom, + preemptionContainerIds, null, null, + application.pullUpdatedNMTokens()); } /** * Process a heartbeat update from a node. */ - private synchronized void nodeUpdate(RMNode nm) { - long start = getClock().getTime(); - if (LOG.isDebugEnabled()) { - LOG.debug("nodeUpdate: " + nm + - " cluster capacity: " + getClusterResource()); - } - eventLog.log("HEARTBEAT", nm.getHostName()); - FSSchedulerNode node = getFSSchedulerNode(nm.getNodeID()); - - List containerInfoList = nm.pullContainerUpdates(); - List newlyLaunchedContainers = new ArrayList(); - List completedContainers = new ArrayList(); - for(UpdatedContainerInfo containerInfo : containerInfoList) { - newlyLaunchedContainers.addAll(containerInfo.getNewlyLaunchedContainers()); - completedContainers.addAll(containerInfo.getCompletedContainers()); - } - // Processing the newly launched containers - for (ContainerStatus launchedContainer : newlyLaunchedContainers) { - containerLaunchedOnNode(launchedContainer.getContainerId(), node); - } + private void nodeUpdate(RMNode nm) { + try { + writeLock.lock(); + long start = getClock().getTime(); + if (LOG.isDebugEnabled()) { + LOG.debug( + "nodeUpdate: " + nm + " cluster capacity: " + getClusterResource()); + } + eventLog.log("HEARTBEAT", nm.getHostName()); + FSSchedulerNode node = getFSSchedulerNode(nm.getNodeID()); - // Process completed containers - for (ContainerStatus completedContainer : completedContainers) { - ContainerId containerId = completedContainer.getContainerId(); - LOG.debug("Container FINISHED: " + containerId); - super.completedContainer(getRMContainer(containerId), - completedContainer, RMContainerEventType.FINISHED); - } + List containerInfoList = nm.pullContainerUpdates(); + List newlyLaunchedContainers = + new ArrayList(); + List completedContainers = + new ArrayList(); + for (UpdatedContainerInfo containerInfo : containerInfoList) { + newlyLaunchedContainers.addAll( + containerInfo.getNewlyLaunchedContainers()); + completedContainers.addAll(containerInfo.getCompletedContainers()); + } + // Processing the newly launched containers + for (ContainerStatus launchedContainer : newlyLaunchedContainers) { + containerLaunchedOnNode(launchedContainer.getContainerId(), node); + } - // If the node is decommissioning, send an update to have the total - // resource equal to the used resource, so no available resource to - // schedule. - if (nm.getState() == NodeState.DECOMMISSIONING) { - this.rmContext - .getDispatcher() - .getEventHandler() - .handle( - new RMNodeResourceUpdateEvent(nm.getNodeID(), ResourceOption - .newInstance(getSchedulerNode(nm.getNodeID()) - .getAllocatedResource(), 0))); - } + // Process completed containers + for (ContainerStatus completedContainer : completedContainers) { + ContainerId containerId = completedContainer.getContainerId(); + LOG.debug("Container FINISHED: " + containerId); + super.completedContainer(getRMContainer(containerId), + completedContainer, RMContainerEventType.FINISHED); + } - if (continuousSchedulingEnabled) { - if (!completedContainers.isEmpty()) { + // If the node is decommissioning, send an update to have the total + // resource equal to the used resource, so no available resource to + // schedule. + if (nm.getState() == NodeState.DECOMMISSIONING) { + this.rmContext.getDispatcher().getEventHandler().handle( + new RMNodeResourceUpdateEvent(nm.getNodeID(), ResourceOption + .newInstance( + getSchedulerNode(nm.getNodeID()).getAllocatedResource(), + 0))); + } + + if (continuousSchedulingEnabled) { + if (!completedContainers.isEmpty()) { + attemptScheduling(node); + } + } else{ attemptScheduling(node); } - } else { - attemptScheduling(node); + + // Updating node resource utilization + node.setAggregatedContainersUtilization( + nm.getAggregatedContainersUtilization()); + node.setNodeUtilization(nm.getNodeUtilization()); + + long duration = getClock().getTime() - start; + fsOpDurations.addNodeUpdateDuration(duration); + } finally { + writeLock.unlock(); } - - // Updating node resource utilization - node.setAggregatedContainersUtilization( - nm.getAggregatedContainersUtilization()); - node.setNodeUtilization(nm.getNodeUtilization()); - - long duration = getClock().getTime() - start; - fsOpDurations.addNodeUpdateDuration(duration); } void continuousSchedulingAttempt() throws InterruptedException { @@ -1125,52 +1174,59 @@ public class FairScheduler extends } @VisibleForTesting - synchronized void attemptScheduling(FSSchedulerNode node) { - if (rmContext.isWorkPreservingRecoveryEnabled() - && !rmContext.isSchedulerReadyForAllocatingContainers()) { - return; - } + void attemptScheduling(FSSchedulerNode node) { + try { + writeLock.lock(); + if (rmContext.isWorkPreservingRecoveryEnabled() && !rmContext + .isSchedulerReadyForAllocatingContainers()) { + return; + } - final NodeId nodeID = node.getNodeID(); - if (!nodeTracker.exists(nodeID)) { - // The node might have just been removed while this thread was waiting - // on the synchronized lock before it entered this synchronized method - LOG.info("Skipping scheduling as the node " + nodeID + - " has been removed"); - return; - } + final NodeId nodeID = node.getNodeID(); + if (!nodeTracker.exists(nodeID)) { + // The node might have just been removed while this thread was waiting + // on the synchronized lock before it entered this synchronized method + LOG.info( + "Skipping scheduling as the node " + nodeID + " has been removed"); + return; + } - // Assign new containers... - // 1. Check for reserved applications - // 2. Schedule if there are no reservations + // Assign new containers... + // 1. Check for reserved applications + // 2. Schedule if there are no reservations - boolean validReservation = false; - FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable(); - if (reservedAppSchedulable != null) { - validReservation = reservedAppSchedulable.assignReservedContainer(node); - } - if (!validReservation) { - // No reservation, schedule at queue which is farthest below fair share - int assignedContainers = 0; - Resource assignedResource = Resources.clone(Resources.none()); - Resource maxResourcesToAssign = - Resources.multiply(node.getUnallocatedResource(), 0.5f); - while (node.getReservedContainer() == null) { - boolean assignedContainer = false; - Resource assignment = queueMgr.getRootQueue().assignContainer(node); - if (!assignment.equals(Resources.none())) { - assignedContainers++; - assignedContainer = true; - Resources.addTo(assignedResource, assignment); - } - if (!assignedContainer) { break; } - if (!shouldContinueAssigning(assignedContainers, - maxResourcesToAssign, assignedResource)) { - break; + boolean validReservation = false; + FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable(); + if (reservedAppSchedulable != null) { + validReservation = reservedAppSchedulable.assignReservedContainer(node); + } + if (!validReservation) { + // No reservation, schedule at queue which is farthest below fair share + int assignedContainers = 0; + Resource assignedResource = Resources.clone(Resources.none()); + Resource maxResourcesToAssign = Resources.multiply( + node.getUnallocatedResource(), 0.5f); + while (node.getReservedContainer() == null) { + boolean assignedContainer = false; + Resource assignment = queueMgr.getRootQueue().assignContainer(node); + if (!assignment.equals(Resources.none())) { + assignedContainers++; + assignedContainer = true; + Resources.addTo(assignedResource, assignment); + } + if (!assignedContainer) { + break; + } + if (!shouldContinueAssigning(assignedContainers, maxResourcesToAssign, + assignedResource)) { + break; + } } } + updateRootQueueMetrics(); + } finally { + writeLock.unlock(); } - updateRootQueueMetrics(); } public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) { @@ -1313,51 +1369,55 @@ public class FairScheduler extends } } - private synchronized String resolveReservationQueueName(String queueName, + private String resolveReservationQueueName(String queueName, ApplicationId applicationId, ReservationId reservationID, boolean isRecovering) { - FSQueue queue = queueMgr.getQueue(queueName); - if ((queue == null) || !allocConf.isReservable(queue.getQueueName())) { - return queueName; - } - // Use fully specified name from now on (including root. prefix) - queueName = queue.getQueueName(); - if (reservationID != null) { - String resQName = queueName + "." + reservationID.toString(); - queue = queueMgr.getQueue(resQName); - if (queue == null) { - // reservation has terminated during failover - if (isRecovering && allocConf.getMoveOnExpiry(queueName)) { - // move to the default child queue of the plan - return getDefaultQueueForPlanQueue(queueName); + try { + readLock.lock(); + FSQueue queue = queueMgr.getQueue(queueName); + if ((queue == null) || !allocConf.isReservable(queue.getQueueName())) { + return queueName; + } + // Use fully specified name from now on (including root. prefix) + queueName = queue.getQueueName(); + if (reservationID != null) { + String resQName = queueName + "." + reservationID.toString(); + queue = queueMgr.getQueue(resQName); + if (queue == null) { + // reservation has terminated during failover + if (isRecovering && allocConf.getMoveOnExpiry(queueName)) { + // move to the default child queue of the plan + return getDefaultQueueForPlanQueue(queueName); + } + String message = "Application " + applicationId + + " submitted to a reservation which is not yet " + + "currently active: " + resQName; + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED, + message)); + return null; } - String message = - "Application " - + applicationId - + " submitted to a reservation which is not yet currently active: " - + resQName; - this.rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, - RMAppEventType.APP_REJECTED, message)); - return null; + if (!queue.getParent().getQueueName().equals(queueName)) { + String message = + "Application: " + applicationId + " submitted to a reservation " + + resQName + " which does not belong to the specified queue: " + + queueName; + this.rmContext.getDispatcher().getEventHandler().handle( + new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED, + message)); + return null; + } + // use the reservation queue to run the app + queueName = resQName; + } else{ + // use the default child queue of the plan for unreserved apps + queueName = getDefaultQueueForPlanQueue(queueName); } - if (!queue.getParent().getQueueName().equals(queueName)) { - String message = - "Application: " + applicationId + " submitted to a reservation " - + resQName + " which does not belong to the specified queue: " - + queueName; - this.rmContext.getDispatcher().getEventHandler() - .handle(new RMAppEvent(applicationId, - RMAppEventType.APP_REJECTED, message)); - return null; - } - // use the reservation queue to run the app - queueName = resQName; - } else { - // use the default child queue of the plan for unreserved apps - queueName = getDefaultQueueForPlanQueue(queueName); + return queueName; + } finally { + readLock.unlock(); } - return queueName; + } private String getDefaultQueueForPlanQueue(String queueName) { @@ -1371,12 +1431,13 @@ public class FairScheduler extends // NOT IMPLEMENTED } - public synchronized void setRMContext(RMContext rmContext) { + public void setRMContext(RMContext rmContext) { this.rmContext = rmContext; } private void initScheduler(Configuration conf) throws IOException { - synchronized (this) { + try { + writeLock.lock(); this.conf = new FairSchedulerConfiguration(conf); validateConf(this.conf); minimumAllocation = this.conf.getMinimumAllocation(); @@ -1384,8 +1445,7 @@ public class FairScheduler extends incrAllocation = this.conf.getIncrementAllocation(); updateReservationThreshold(); continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled(); - continuousSchedulingSleepMs = - this.conf.getContinuousSchedulingSleepMs(); + continuousSchedulingSleepMs = this.conf.getContinuousSchedulingSleepMs(); nodeLocalityThreshold = this.conf.getLocalityThresholdNode(); rackLocalityThreshold = this.conf.getLocalityThresholdRack(); nodeLocalityDelayMs = this.conf.getLocalityDelayNodeMs(); @@ -1406,8 +1466,8 @@ public class FairScheduler extends if (updateInterval < 0) { updateInterval = FairSchedulerConfiguration.DEFAULT_UPDATE_INTERVAL_MS; LOG.warn(FairSchedulerConfiguration.UPDATE_INTERVAL_MS - + " is invalid, so using default value " + - +FairSchedulerConfiguration.DEFAULT_UPDATE_INTERVAL_MS + + " is invalid, so using default value " + + +FairSchedulerConfiguration.DEFAULT_UPDATE_INTERVAL_MS + " ms instead"); } @@ -1415,8 +1475,7 @@ public class FairScheduler extends fsOpDurations = FSOpDurations.getInstance(true); // This stores per-application scheduling information - this.applications = new ConcurrentHashMap< - ApplicationId, SchedulerApplication>(); + this.applications = new ConcurrentHashMap<>(); this.eventLog = new FairSchedulerEventLog(); eventLog.init(this.conf); @@ -1437,6 +1496,8 @@ public class FairScheduler extends schedulingThread.setName("FairSchedulerContinuousScheduling"); schedulingThread.setDaemon(true); } + } finally { + writeLock.unlock(); } allocsLoader.init(conf); @@ -1459,15 +1520,21 @@ public class FairScheduler extends reservationThreshold = newThreshold; } - private synchronized void startSchedulerThreads() { - Preconditions.checkNotNull(updateThread, "updateThread is null"); - Preconditions.checkNotNull(allocsLoader, "allocsLoader is null"); - updateThread.start(); - if (continuousSchedulingEnabled) { - Preconditions.checkNotNull(schedulingThread, "schedulingThread is null"); - schedulingThread.start(); + private void startSchedulerThreads() { + try { + writeLock.lock(); + Preconditions.checkNotNull(updateThread, "updateThread is null"); + Preconditions.checkNotNull(allocsLoader, "allocsLoader is null"); + updateThread.start(); + if (continuousSchedulingEnabled) { + Preconditions.checkNotNull(schedulingThread, + "schedulingThread is null"); + schedulingThread.start(); + } + allocsLoader.start(); + } finally { + writeLock.unlock(); } - allocsLoader.start(); } @Override @@ -1484,7 +1551,8 @@ public class FairScheduler extends @Override public void serviceStop() throws Exception { - synchronized (this) { + try { + writeLock.lock(); if (updateThread != null) { updateThread.interrupt(); updateThread.join(THREAD_JOIN_TIMEOUT_MS); @@ -1498,6 +1566,8 @@ public class FairScheduler extends if (allocsLoader != null) { allocsLoader.stop(); } + } finally { + writeLock.unlock(); } super.serviceStop(); @@ -1541,17 +1611,22 @@ public class FairScheduler extends } @Override - public synchronized boolean checkAccess(UserGroupInformation callerUGI, + public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, String queueName) { - FSQueue queue = getQueueManager().getQueue(queueName); - if (queue == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("ACL not found for queue access-type " + acl - + " for queue " + queueName); + try { + readLock.lock(); + FSQueue queue = getQueueManager().getQueue(queueName); + if (queue == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("ACL not found for queue access-type " + acl + " for queue " + + queueName); + } + return false; } - return false; + return queue.hasAccess(acl, callerUGI); + } finally { + readLock.unlock(); } - return queue.hasAccess(acl, callerUGI); } public AllocationConfiguration getAllocationConfiguration() { @@ -1565,12 +1640,16 @@ public class FairScheduler extends public void onReload(AllocationConfiguration queueInfo) { // Commit the reload; also create any queue defined in the alloc file // if it does not already exist, so it can be displayed on the web UI. - synchronized (FairScheduler.this) { + + writeLock.lock(); + try { allocConf = queueInfo; allocConf.getDefaultSchedulingPolicy().initialize(getClusterResource()); queueMgr.updateAllocationConfiguration(allocConf); applyChildDefaults(); maxRunningEnforcer.updateRunnabilityOnReload(); + } finally { + writeLock.unlock(); } } } @@ -1615,32 +1694,41 @@ public class FairScheduler extends } @Override - public synchronized String moveApplication(ApplicationId appId, + public String moveApplication(ApplicationId appId, String queueName) throws YarnException { - SchedulerApplication app = applications.get(appId); - if (app == null) { - throw new YarnException("App to be moved " + appId + " not found."); - } - FSAppAttempt attempt = (FSAppAttempt) app.getCurrentAppAttempt(); - // To serialize with FairScheduler#allocate, synchronize on app attempt - synchronized (attempt) { - FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); - String destQueueName = handleMoveToPlanQueue(queueName); - FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false); - if (targetQueue == null) { - throw new YarnException("Target queue " + queueName - + " not found or is not a leaf queue."); + try { + writeLock.lock(); + SchedulerApplication app = applications.get(appId); + if (app == null) { + throw new YarnException("App to be moved " + appId + " not found."); } - if (targetQueue == oldQueue) { - return oldQueue.getQueueName(); + FSAppAttempt attempt = (FSAppAttempt) app.getCurrentAppAttempt(); + // To serialize with FairScheduler#allocate, synchronize on app attempt + + try { + attempt.getWriteLock().lock(); + FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); + String destQueueName = handleMoveToPlanQueue(queueName); + FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false); + if (targetQueue == null) { + throw new YarnException("Target queue " + queueName + + " not found or is not a leaf queue."); + } + if (targetQueue == oldQueue) { + return oldQueue.getQueueName(); + } + + if (oldQueue.isRunnableApp(attempt)) { + verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue); + } + + executeMove(app, attempt, oldQueue, targetQueue); + return targetQueue.getQueueName(); + } finally { + attempt.getWriteLock().unlock(); } - - if (oldQueue.isRunnableApp(attempt)) { - verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue); - } - - executeMove(app, attempt, oldQueue, targetQueue); - return targetQueue.getQueueName(); + } finally { + writeLock.unlock(); } } @@ -1736,12 +1824,17 @@ public class FairScheduler extends * Process resource update on a node and update Queue. */ @Override - public synchronized void updateNodeResource(RMNode nm, + public void updateNodeResource(RMNode nm, ResourceOption resourceOption) { - super.updateNodeResource(nm, resourceOption); - updateRootQueueMetrics(); - queueMgr.getRootQueue().setSteadyFairShare(getClusterResource()); - queueMgr.getRootQueue().recomputeSteadyShares(); + try { + writeLock.lock(); + super.updateNodeResource(nm, resourceOption); + updateRootQueueMetrics(); + queueMgr.getRootQueue().setSteadyFairShare(getClusterResource()); + queueMgr.getRootQueue().recomputeSteadyShares(); + } finally { + writeLock.unlock(); + } } /** {@inheritDoc} */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ErrorBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ErrorBlock.java new file mode 100644 index 00000000000..963e53f8037 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ErrorBlock.java @@ -0,0 +1,39 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; + +import com.google.inject.Inject; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.ERROR_MESSAGE; + +/** + * This class is used to display an error message to the user in the UI. + */ +public class ErrorBlock extends HtmlBlock { + @Inject + ErrorBlock(ViewContext ctx) { + super(ctx); + } + + @Override + protected void render(Block html) { + html.p()._($(ERROR_MESSAGE))._(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 106065bac81..2d7139f228e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -71,6 +71,7 @@ public class RMWebApp extends WebApp implements YarnWebParams { route("/errors-and-warnings", RmController.class, "errorsAndWarnings"); route(pajoin("/logaggregationstatus", APPLICATION_ID), RmController.class, "logaggregationstatus"); + route(pajoin("/failure", APPLICATION_ID), RmController.class, "failure"); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RedirectionErrorPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RedirectionErrorPage.java new file mode 100644 index 00000000000..beb0cca235d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RedirectionErrorPage.java @@ -0,0 +1,47 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.YarnWebParams; + +/** + * This class is used to display a message that the proxy request failed + * because of a redirection issue. + */ +public class RedirectionErrorPage extends RmView { + @Override protected void preHead(Page.HTML<_> html) { + String aid = $(YarnWebParams.APPLICATION_ID); + + commonPreHead(html); + set(YarnWebParams.ERROR_MESSAGE, + "The application master for " + aid + " redirected the " + + "resource manager's web proxy's request back to the web proxy, " + + "which means your request to view the application master's web UI " + + "cannot be fulfilled. The typical cause for this error is a " + + "network misconfiguration that causes the resource manager's web " + + "proxy host to resolve to an unexpected IP address on the " + + "application master host. Please contact your cluster " + + "administrator to resolve the issue."); + } + + @Override protected Class content() { + return ErrorBlock.class; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java index b124d7585a4..a291e0548db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java @@ -62,6 +62,10 @@ public class RmController extends Controller { render(ContainerPage.class); } + public void failure() { + render(RedirectionErrorPage.class); + } + public void nodes() { render(NodesPage.class); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java index 71ee924e3d3..42a07af6564 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java @@ -44,6 +44,9 @@ public class ReservationDefinitionInfo { @XmlElement(name = "reservation-name") private String reservationName; + @XmlElement(name = "priority") + private int priority; + public ReservationDefinitionInfo() { } @@ -89,4 +92,12 @@ public class ReservationDefinitionInfo { this.reservationName = reservationName; } + public int getPriority() { + return priority; + } + + public void setPriority(int priority) { + this.priority = priority; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java index 07c6b54f1fb..207f5ba0e38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistributedSche import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterDistributedSchedulingAMResponsePBImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.junit.Assert; import org.junit.Test; @@ -97,6 +98,11 @@ public class TestOpportunisticContainerAllocatorAMService { public Configuration getYarnConfiguration() { return new YarnConfiguration(); } + + @Override + public RMContainerTokenSecretManager getContainerTokenSecretManager() { + return new RMContainerTokenSecretManager(conf); + } }; Container c = factory.newRecordInstance(Container.class); c.setExecutionType(ExecutionType.OPPORTUNISTIC); @@ -117,8 +123,8 @@ public class TestOpportunisticContainerAllocatorAMService { Server server = service.getServer(rpc, conf, addr, null); server.start(); - // Verify that the DistrubutedSchedulingService can handle vanilla - // ApplicationMasterProtocol clients + // Verify that the OpportunisticContainerAllocatorAMSercvice can handle + // vanilla ApplicationMasterProtocol clients RPC.setProtocolEngine(conf, ApplicationMasterProtocolPB.class, ProtobufRpcEngine.class); ApplicationMasterProtocolPB ampProxy = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 0b65c0b50c9..a3022f7a487 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.hadoop.conf.Configuration; @@ -64,9 +65,9 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -1085,6 +1086,106 @@ public class TestRMAdminService { rm.close(); } + @Test + public void testModifyLabelsOnUnknownNodes() throws IOException, + YarnException { + // create RM and set it's ACTIVE, and set distributed node label + // configuration to true + rm = new MockRM(); + + ((RMContextImpl) rm.getRMContext()) + .setHAServiceState(HAServiceState.ACTIVE); + Map rmNodes = rm.getRMContext().getRMNodes(); + rmNodes.put(NodeId.newInstance("host1", 1111), + new RMNodeImpl(null, rm.getRMContext(), "host1", 0, 0, null, null, + null)); + rmNodes.put(NodeId.newInstance("host2", 2222), + new RMNodeImpl(null, rm.getRMContext(), "host2", 0, 0, null, null, + null)); + rmNodes.put(NodeId.newInstance("host3", 3333), + new RMNodeImpl(null, rm.getRMContext(), "host3", 0, 0, null, null, + null)); + Map rmInactiveNodes = rm.getRMContext() + .getInactiveRMNodes(); + rmInactiveNodes.put(NodeId.newInstance("host4", 4444), + new RMNodeImpl(null, rm.getRMContext(), "host4", 0, 0, null, null, + null)); + RMNodeLabelsManager labelMgr = rm.rmContext.getNodeLabelManager(); + + // by default, distributed configuration for node label is disabled, this + // should pass + labelMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", + "y")); + // replace known node + ReplaceLabelsOnNodeRequest request1 = ReplaceLabelsOnNodeRequest + .newInstance(ImmutableMap.of(NodeId.newInstance("host1", 1111), + (Set) ImmutableSet.of("x"))); + request1.setFailOnUnknownNodes(true); + try { + rm.adminService.replaceLabelsOnNode(request1); + } catch (Exception ex) { + fail("should not fail on known node"); + } + + // replace known node with wildcard port + ReplaceLabelsOnNodeRequest request2 = ReplaceLabelsOnNodeRequest + .newInstance(ImmutableMap.of(NodeId.newInstance("host1", 0), + (Set) ImmutableSet.of("x"))); + request2.setFailOnUnknownNodes(true); + try { + rm.adminService.replaceLabelsOnNode(request2); + } catch (Exception ex) { + fail("should not fail on known node"); + } + + // replace unknown node + ReplaceLabelsOnNodeRequest request3 = ReplaceLabelsOnNodeRequest + .newInstance(ImmutableMap.of(NodeId.newInstance("host5", 0), + (Set) ImmutableSet.of("x"))); + request3.setFailOnUnknownNodes(true); + try { + rm.adminService.replaceLabelsOnNode(request3); + fail("Should fail on unknown node"); + } catch (Exception ex) { + } + + // replace known node but wrong port + ReplaceLabelsOnNodeRequest request4 = ReplaceLabelsOnNodeRequest + .newInstance(ImmutableMap.of(NodeId.newInstance("host2", 1111), + (Set) ImmutableSet.of("x"))); + request4.setFailOnUnknownNodes(true); + try { + rm.adminService.replaceLabelsOnNode(request4); + fail("Should fail on node with wrong port"); + } catch (Exception ex) { + } + + // replace non-exist node but not check + ReplaceLabelsOnNodeRequest request5 = ReplaceLabelsOnNodeRequest + .newInstance(ImmutableMap.of(NodeId.newInstance("host5", 0), + (Set) ImmutableSet.of("x"))); + request5.setFailOnUnknownNodes(false); + try { + rm.adminService.replaceLabelsOnNode(request5); + } catch (Exception ex) { + fail("Should not fail on unknown node when " + + "fail-on-unkown-nodes is set false"); + } + + // replace on inactive node + ReplaceLabelsOnNodeRequest request6 = ReplaceLabelsOnNodeRequest + .newInstance(ImmutableMap.of(NodeId.newInstance("host4", 0), + (Set) ImmutableSet.of("x"))); + request6.setFailOnUnknownNodes(true); + try { + rm.adminService.replaceLabelsOnNode(request6); + } catch (Exception ex) { + fail("should not fail on inactive node"); + } + + rm.close(); + } + @Test public void testRemoveClusterNodeLabelsWithCentralizedConfigurationDisabled() throws IOException, YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java index 20b1c0e0060..bfd0b4e75ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java @@ -28,6 +28,14 @@ import org.junit.Test; import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.atMost; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class TestRMEmbeddedElector extends ClientBaseWithFixes { private static final Log LOG = @@ -41,6 +49,14 @@ public class TestRMEmbeddedElector extends ClientBaseWithFixes { private Configuration conf; private AtomicBoolean callbackCalled; + private enum SyncTestType { + ACTIVE, + STANDBY, + NEUTRAL, + ACTIVE_TIMING, + STANDBY_TIMING + } + @Before public void setup() throws IOException { conf = new YarnConfiguration(); @@ -79,6 +95,181 @@ public class TestRMEmbeddedElector extends ClientBaseWithFixes { LOG.info("Stopped RM"); } + /** + * Test that neutral mode plays well with all other transitions. + * + * @throws IOException if there's an issue transitioning + * @throws InterruptedException if interrupted + */ + @Test + public void testCallbackSynchronization() + throws IOException, InterruptedException { + testCallbackSynchronization(SyncTestType.ACTIVE); + testCallbackSynchronization(SyncTestType.STANDBY); + testCallbackSynchronization(SyncTestType.NEUTRAL); + testCallbackSynchronization(SyncTestType.ACTIVE_TIMING); + testCallbackSynchronization(SyncTestType.STANDBY_TIMING); + } + + /** + * Helper method to test that neutral mode plays well with other transitions. + * + * @param type the type of test to run + * @throws IOException if there's an issue transitioning + * @throws InterruptedException if interrupted + */ + private void testCallbackSynchronization(SyncTestType type) + throws IOException, InterruptedException { + AdminService as = mock(AdminService.class); + RMContext rc = mock(RMContext.class); + Configuration myConf = new Configuration(conf); + + myConf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 50); + when(rc.getRMAdminService()).thenReturn(as); + + EmbeddedElectorService ees = new EmbeddedElectorService(rc); + ees.init(myConf); + + ees.enterNeutralMode(); + + switch (type) { + case ACTIVE: + testCallbackSynchronizationActive(as, ees); + break; + case STANDBY: + testCallbackSynchronizationStandby(as, ees); + break; + case NEUTRAL: + testCallbackSynchronizationNeutral(as, ees); + break; + case ACTIVE_TIMING: + testCallbackSynchronizationTimingActive(as, ees); + break; + case STANDBY_TIMING: + testCallbackSynchronizationTimingStandby(as, ees); + break; + default: + fail("Unknown test type: " + type); + break; + } + } + + /** + * Helper method to test that neutral mode plays well with an active + * transition. + * + * @param as the admin service + * @param ees the embedded elector service + * @throws IOException if there's an issue transitioning + * @throws InterruptedException if interrupted + */ + private void testCallbackSynchronizationActive(AdminService as, + EmbeddedElectorService ees) throws IOException, InterruptedException { + ees.becomeActive(); + + Thread.sleep(100); + + verify(as).transitionToActive(any()); + verify(as, never()).transitionToStandby(any()); + } + + /** + * Helper method to test that neutral mode plays well with a standby + * transition. + * + * @param as the admin service + * @param ees the embedded elector service + * @throws IOException if there's an issue transitioning + * @throws InterruptedException if interrupted + */ + private void testCallbackSynchronizationStandby(AdminService as, + EmbeddedElectorService ees) throws IOException, InterruptedException { + ees.becomeStandby(); + + Thread.sleep(100); + + verify(as, atLeast(1)).transitionToStandby(any()); + verify(as, atMost(1)).transitionToStandby(any()); + } + + /** + * Helper method to test that neutral mode plays well with itself. + * + * @param as the admin service + * @param ees the embedded elector service + * @throws IOException if there's an issue transitioning + * @throws InterruptedException if interrupted + */ + private void testCallbackSynchronizationNeutral(AdminService as, + EmbeddedElectorService ees) throws IOException, InterruptedException { + ees.enterNeutralMode(); + + Thread.sleep(100); + + verify(as, atLeast(1)).transitionToStandby(any()); + verify(as, atMost(1)).transitionToStandby(any()); + } + + /** + * Helper method to test that neutral mode does not race with an active + * transition. + * + * @param as the admin service + * @param ees the embedded elector service + * @throws IOException if there's an issue transitioning + * @throws InterruptedException if interrupted + */ + private void testCallbackSynchronizationTimingActive(AdminService as, + EmbeddedElectorService ees) throws IOException, InterruptedException { + synchronized (ees.zkDisconnectLock) { + // Sleep while holding the lock so that the timer thread can't do + // anything when it runs. Sleep until we're pretty sure the timer thread + // has tried to run. + Thread.sleep(100); + // While still holding the lock cancel the timer by transitioning. This + // simulates a race where the callback goes to cancel the timer while the + // timer is trying to run. + ees.becomeActive(); + } + + // Sleep just a little more so that the timer thread can do whatever it's + // going to do, hopefully nothing. + Thread.sleep(50); + + verify(as).transitionToActive(any()); + verify(as, never()).transitionToStandby(any()); + } + + /** + * Helper method to test that neutral mode does not race with an active + * transition. + * + * @param as the admin service + * @param ees the embedded elector service + * @throws IOException if there's an issue transitioning + * @throws InterruptedException if interrupted + */ + private void testCallbackSynchronizationTimingStandby(AdminService as, + EmbeddedElectorService ees) throws IOException, InterruptedException { + synchronized (ees.zkDisconnectLock) { + // Sleep while holding the lock so that the timer thread can't do + // anything when it runs. Sleep until we're pretty sure the timer thread + // has tried to run. + Thread.sleep(100); + // While still holding the lock cancel the timer by transitioning. This + // simulates a race where the callback goes to cancel the timer while the + // timer is trying to run. + ees.becomeStandby(); + } + + // Sleep just a little more so that the timer thread can do whatever it's + // going to do, hopefully nothing. + Thread.sleep(50); + + verify(as, atLeast(1)).transitionToStandby(any()); + verify(as, atMost(1)).transitionToStandby(any()); + } + private class MockRMWithElector extends MockRM { private long delayMs = 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index 24c386a7170..1ff6a1a8185 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -31,6 +31,7 @@ import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.ReservationRequest; @@ -199,6 +200,13 @@ public class ReservationSystemTestUtil { public static ReservationSubmissionRequest createSimpleReservationRequest( ReservationId reservationId, int numContainers, long arrival, long deadline, long duration) { + return createSimpleReservationRequest(reservationId, numContainers, + arrival, deadline, duration, Priority.UNDEFINED); + } + + public static ReservationSubmissionRequest createSimpleReservationRequest( + ReservationId reservationId, int numContainers, long arrival, + long deadline, long duration, Priority priority) { // create a request with a single atomic ask ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), @@ -208,7 +216,7 @@ public class ReservationSystemTestUtil { ReservationRequestInterpreter.R_ALL); ReservationDefinition rDef = ReservationDefinition.newInstance(arrival, deadline, reqs, - "testClientRMService#reservation"); + "testClientRMService#reservation", "0", priority); ReservationSubmissionRequest request = ReservationSubmissionRequest.newInstance(rDef, reservationQ, reservationId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 5dfee89bc6f..205188b2c02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -1148,17 +1148,21 @@ public class TestDelegationTokenRenewer { credentials, null, true, false, false, null, 0, null, false, null); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING); + DelegationTokenRenewer renewer = + rm.getRMContext().getDelegationTokenRenewer(); + DelegationTokenToRenew dttr = renewer.getAllTokens().get(token1); + Assert.assertNotNull(dttr); // submit app2 with the same token, set cancelTokenWhenComplete to true; RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, true, null); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1); rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING); - MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2); + finishAMAndWaitForComplete(app2, rm, nm1, am2, dttr); Assert.assertTrue(rm.getRMContext().getDelegationTokenRenewer() .getAllTokens().containsKey(token1)); - MockRM.finishAMAndVerifyAppState(app1, rm, nm1, am1); + finishAMAndWaitForComplete(app1, rm, nm1, am1, dttr); // app2 completes, app1 is still running, check the token is not cancelled Assert.assertFalse(Renewer.cancelled); } @@ -1224,7 +1228,7 @@ public class TestDelegationTokenRenewer { Assert.assertTrue(dttr.referringAppIds.contains(app2.getApplicationId())); Assert.assertFalse(Renewer.cancelled); - MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2); + finishAMAndWaitForComplete(app2, rm, nm1, am2, dttr); // app2 completes, app1 is still running, check the token is not cancelled Assert.assertTrue(renewer.getAllTokens().containsKey(token1)); Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId())); @@ -1242,14 +1246,14 @@ public class TestDelegationTokenRenewer { Assert.assertFalse(dttr.isTimerCancelled()); Assert.assertFalse(Renewer.cancelled); - MockRM.finishAMAndVerifyAppState(app1, rm, nm1, am1); + finishAMAndWaitForComplete(app1, rm, nm1, am1, dttr); Assert.assertTrue(renewer.getAllTokens().containsKey(token1)); Assert.assertFalse(dttr.referringAppIds.contains(app1.getApplicationId())); Assert.assertTrue(dttr.referringAppIds.contains(app3.getApplicationId())); Assert.assertFalse(dttr.isTimerCancelled()); Assert.assertFalse(Renewer.cancelled); - MockRM.finishAMAndVerifyAppState(app3, rm, nm1, am3); + finishAMAndWaitForComplete(app3, rm, nm1, am3, dttr); Assert.assertFalse(renewer.getAllTokens().containsKey(token1)); Assert.assertTrue(dttr.referringAppIds.isEmpty()); Assert.assertTrue(dttr.isTimerCancelled()); @@ -1259,4 +1263,14 @@ public class TestDelegationTokenRenewer { Assert.assertFalse(renewer.getDelegationTokens().contains(token1)); } + private void finishAMAndWaitForComplete(final RMApp app, MockRM rm, + MockNM nm, MockAM am, final DelegationTokenToRenew dttr) + throws Exception { + MockRM.finishAMAndVerifyAppState(app, rm, nm, am); + GenericTestUtils.waitFor(new Supplier() { + public Boolean get() { + return !dttr.referringAppIds.contains(app.getApplicationId()); + } + }, 10, 10000); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRedirectionErrorPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRedirectionErrorPage.java new file mode 100644 index 00000000000..408dc9bb88a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRedirectionErrorPage.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + + +import java.io.IOException; + +import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.webapp.YarnWebParams; +import org.apache.hadoop.yarn.webapp.test.WebAppTests; +import org.junit.Test; + +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Module; + +/** + * This class tests the RedirectionErrorPage. + */ +public class TestRedirectionErrorPage { + @Test + public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception { + ApplicationId appId = ApplicationId.newInstance(1234L, 0); + Injector injector; + + // initialize RM Context, and create RMApp, without creating RMAppAttempt + final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8); + + injector = WebAppTests.createMockInjector(RMContext.class, rmContext, + new Module() { + @Override + public void configure(Binder binder) { + try { + ResourceManager rm = TestRMWebApp.mockRm(rmContext); + binder.bind(ResourceManager.class).toInstance(rm); + binder.bind(ApplicationBaseProtocol.class).toInstance( + rm.getClientRMService()); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + }); + + ErrorBlock instance = injector.getInstance(ErrorBlock.class); + instance.set(YarnWebParams.APPLICATION_ID, appId.toString()); + instance.set(YarnWebParams.ERROR_MESSAGE, "This is an error"); + instance.render(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java index e130225c892..c6567428ea6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java @@ -40,6 +40,8 @@ public class ProxyUriUtils { public static final String PROXY_SERVLET_NAME = "proxy"; /**Base path where the proxy servlet will handle requests.*/ public static final String PROXY_BASE = "/proxy/"; + /**Path component added when the proxy redirects the connection.*/ + public static final String REDIRECT = "redirect/"; /**Path Specification for the proxy servlet.*/ public static final String PROXY_PATH_SPEC = PROXY_BASE+"*"; /**Query Parameter indicating that the URI was approved.*/ @@ -57,27 +59,58 @@ public class ProxyUriUtils { /** * Get the proxied path for an application. - * @param id the application id to use. - * @return the base path to that application through the proxy. + * + * @param id the application id to use + * @return the base path to that application through the proxy */ public static String getPath(ApplicationId id) { - if(id == null) { - throw new IllegalArgumentException("Application id cannot be null "); - } - return ujoin(PROXY_BASE, uriEncode(id)); + return getPath(id, false); } /** * Get the proxied path for an application. - * @param id the application id to use. - * @param path the rest of the path to the application. - * @return the base path to that application through the proxy. + * + * @param id the application id to use + * @param redirected whether the path should contain the redirect component + * @return the base path to that application through the proxy + */ + public static String getPath(ApplicationId id, boolean redirected) { + if (id == null) { + throw new IllegalArgumentException("Application id cannot be null "); + } + + if (redirected) { + return ujoin(PROXY_BASE, REDIRECT, uriEncode(id)); + } else { + return ujoin(PROXY_BASE, uriEncode(id)); + } + } + + /** + * Get the proxied path for an application. + * + * @param id the application id to use + * @param path the rest of the path to the application + * @return the base path to that application through the proxy */ public static String getPath(ApplicationId id, String path) { - if(path == null) { - return getPath(id); + return getPath(id, path, false); + } + + /** + * Get the proxied path for an application. + * + * @param id the application id to use + * @param path the rest of the path to the application + * @param redirected whether the path should contain the redirect component + * @return the base path to that application through the proxy + */ + public static String getPath(ApplicationId id, String path, + boolean redirected) { + if (path == null) { + return getPath(id, redirected); } else { - return ujoin(getPath(id), path); + return ujoin(getPath(id, redirected), path); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java index 0b621aa182a..b32ee301554 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java @@ -26,6 +26,7 @@ import java.io.ObjectInputStream; import java.io.OutputStream; import java.io.PrintWriter; import java.net.InetAddress; +import java.net.SocketException; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; @@ -42,8 +43,10 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.UriBuilder; +import javax.ws.rs.core.UriBuilderException; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -76,7 +79,8 @@ public class WebAppProxyServlet extends HttpServlet { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger( WebAppProxyServlet.class); - private static final Set passThroughHeaders = + private static final String REDIRECT = "/redirect"; + private static final Set PASS_THROUGH_HEADERS = new HashSet<>(Arrays.asList( "User-Agent", "Accept", @@ -93,6 +97,7 @@ public class WebAppProxyServlet extends HttpServlet { private transient List trackingUriPlugins; private final String rmAppPageUrlBase; private final String ahsAppPageUrlBase; + private final String failurePageUrlBase; private transient YarnConfiguration conf; /** @@ -126,11 +131,16 @@ public class WebAppProxyServlet extends HttpServlet { this.trackingUriPlugins = conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR, TrackingUriPlugin.class); - this.rmAppPageUrlBase = StringHelper.pjoin( - WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), "cluster", "app"); - this.ahsAppPageUrlBase = StringHelper.pjoin( - WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils - .getAHSWebAppURLWithoutScheme(conf), "applicationhistory", "app"); + this.rmAppPageUrlBase = + StringHelper.pjoin(WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), + "cluster", "app"); + this.failurePageUrlBase = + StringHelper.pjoin(WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), + "cluster", "failure"); + this.ahsAppPageUrlBase = + StringHelper.pjoin(WebAppUtils.getHttpSchemePrefix(conf) + + WebAppUtils.getAHSWebAppURLWithoutScheme(conf), + "applicationhistory", "app"); } /** @@ -220,9 +230,9 @@ public class WebAppProxyServlet extends HttpServlet { @SuppressWarnings("unchecked") Enumeration names = req.getHeaderNames(); - while(names.hasMoreElements()) { + while (names.hasMoreElements()) { String name = names.nextElement(); - if(passThroughHeaders.contains(name)) { + if (PASS_THROUGH_HEADERS.contains(name)) { String value = req.getHeader(name); if (LOG.isDebugEnabled()) { LOG.debug("REQ HEADER: {} : {}", name, value); @@ -312,30 +322,49 @@ public class WebAppProxyServlet extends HttpServlet { boolean userWasWarned = false; boolean userApproved = Boolean.parseBoolean(userApprovedParamS); boolean securityEnabled = isSecurityEnabled(); + boolean isRedirect = false; + String pathInfo = req.getPathInfo(); final String remoteUser = req.getRemoteUser(); - final String pathInfo = req.getPathInfo(); String[] parts = null; + if (pathInfo != null) { + // If there's a redirect, strip the redirect so that the path can be + // parsed + if (pathInfo.startsWith(REDIRECT)) { + pathInfo = pathInfo.substring(REDIRECT.length()); + isRedirect = true; + } + parts = pathInfo.split("/", 3); } - if(parts == null || parts.length < 2) { + + if ((parts == null) || (parts.length < 2)) { LOG.warn("{} gave an invalid proxy path {}", remoteUser, pathInfo); notFound(resp, "Your path appears to be formatted incorrectly."); return; } + //parts[0] is empty because path info always starts with a / String appId = parts[1]; String rest = parts.length > 2 ? parts[2] : ""; ApplicationId id = Apps.toAppID(appId); - if(id == null) { + + if (id == null) { LOG.warn("{} attempting to access {} that is invalid", remoteUser, appId); notFound(resp, appId + " appears to be formatted incorrectly."); return; } - - if(securityEnabled) { + + // If this call is from an AM redirect, we need to be careful about how + // we handle it. If this method returns true, it means the method + // already redirected the response, so we can just return. + if (isRedirect && handleRedirect(appId, req, resp)) { + return; + } + + if (securityEnabled) { String cookieName = getCheckCookieName(id); Cookie[] cookies = req.getCookies(); if (cookies != null) { @@ -351,22 +380,21 @@ public class WebAppProxyServlet extends HttpServlet { boolean checkUser = securityEnabled && (!userWasWarned || !userApproved); - FetchedAppReport fetchedAppReport = null; - ApplicationReport applicationReport = null; + FetchedAppReport fetchedAppReport; + try { - fetchedAppReport = getApplicationReport(id); - if (fetchedAppReport != null) { - if (fetchedAppReport.getAppReportSource() != AppReportSource.RM && - fetchedAppReport.getAppReportSource() != AppReportSource.AHS) { - throw new UnsupportedOperationException("Application report not " - + "fetched from RM or history server."); - } - applicationReport = fetchedAppReport.getApplicationReport(); - } + fetchedAppReport = getFetchedAppReport(id); } catch (ApplicationNotFoundException e) { - applicationReport = null; + fetchedAppReport = null; } - if(applicationReport == null) { + + ApplicationReport applicationReport = null; + + if (fetchedAppReport != null) { + applicationReport = fetchedAppReport.getApplicationReport(); + } + + if (applicationReport == null) { LOG.warn("{} attempting to access {} that was not found", remoteUser, id); @@ -382,57 +410,31 @@ public class WebAppProxyServlet extends HttpServlet { "in RM or history server"); return; } - String original = applicationReport.getOriginalTrackingUrl(); - URI trackingUri; - if (original == null || original.equals("N/A") || original.equals("")) { - if (fetchedAppReport.getAppReportSource() == AppReportSource.RM) { - // fallback to ResourceManager's app page if no tracking URI provided - // and Application Report was fetched from RM - LOG.debug("Original tracking url is '{}'. Redirecting to RM app page", - original == null? "NULL" : original); - ProxyUtils.sendRedirect(req, resp, - StringHelper.pjoin(rmAppPageUrlBase, id.toString())); - } else if (fetchedAppReport.getAppReportSource() - == AppReportSource.AHS) { - // fallback to Application History Server app page if the application - // report was fetched from AHS - LOG.debug("Original tracking url is '{}'. Redirecting to AHS app page" - , original == null? "NULL" : original); - ProxyUtils.sendRedirect(req, resp, - StringHelper.pjoin(ahsAppPageUrlBase, id.toString())); - } + + URI trackingUri = getTrackingUri(req, resp, id, + applicationReport.getOriginalTrackingUrl(), + fetchedAppReport.getAppReportSource()); + + // If the tracking URI is null, there was a redirect, so just return. + if (trackingUri == null) { return; - } else { - if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) { - trackingUri = ProxyUriUtils.getUriFromAMUrl( - WebAppUtils.getHttpSchemePrefix(conf), original); - } else { - trackingUri = new URI(original); - } } String runningUser = applicationReport.getUser(); - if(checkUser && !runningUser.equals(remoteUser)) { + + if (checkUser && !runningUser.equals(remoteUser)) { LOG.info("Asking {} if they want to connect to the " + "app master GUI of {} owned by {}", remoteUser, appId, runningUser); warnUserPage(resp, ProxyUriUtils.getPathAndQuery(id, rest, req.getQueryString(), true), runningUser, id); + return; } // Append the user-provided path and query parameter to the original // tracking url. - UriBuilder builder = UriBuilder.fromUri(trackingUri); - String queryString = req.getQueryString(); - if (queryString != null) { - List queryPairs = - URLEncodedUtils.parse(queryString, null); - for (NameValuePair pair : queryPairs) { - builder.queryParam(pair.getName(), pair.getValue()); - } - } - URI toFetch = builder.path(rest).build(); + URI toFetch = buildTrackingUrl(trackingUri, req, rest); LOG.info("{} is accessing unchecked {}" + " which is the app master GUI of {} owned by {}", @@ -458,6 +460,152 @@ public class WebAppProxyServlet extends HttpServlet { } } + /** + * Return a URL based on the {@code trackingUri} that includes the + * user-provided path and query parameters. + * + * @param trackingUri the base tracking URI + * @param req the service request + * @param rest the user-provided path + * @return the new tracking URI + * @throws UriBuilderException if there's an error building the URL + */ + private URI buildTrackingUrl(URI trackingUri, final HttpServletRequest req, + String rest) throws UriBuilderException { + UriBuilder builder = UriBuilder.fromUri(trackingUri); + String queryString = req.getQueryString(); + + if (queryString != null) { + List queryPairs = URLEncodedUtils.parse(queryString, null); + + for (NameValuePair pair : queryPairs) { + builder.queryParam(pair.getName(), pair.getValue()); + } + } + + return builder.path(rest).build(); + } + + /** + * Locate the tracking URI for the application based on the reported tracking + * URI. If the reported URI is invalid, redirect to the history server or RM + * app page. If the URI is valid, covert it into a usable URI object with a + * schema. If the returned URI is null, that means there was a redirect. + * + * @param req the servlet request for redirects + * @param resp the servlet response for redirects + * @param id the application ID + * @param originalUri the reported tracking URI + * @param appReportSource the source of the application report + * @return a valid tracking URI or null if redirected instead + * @throws IOException thrown if the redirect fails + * @throws URISyntaxException if the tracking URI is invalid + */ + private URI getTrackingUri(HttpServletRequest req, HttpServletResponse resp, + ApplicationId id, String originalUri, AppReportSource appReportSource) + throws IOException, URISyntaxException { + URI trackingUri = null; + + if ((originalUri == null) || + originalUri.equals("N/A") || + originalUri.equals("")) { + if (appReportSource == AppReportSource.RM) { + // fallback to ResourceManager's app page if no tracking URI provided + // and Application Report was fetched from RM + LOG.debug("Original tracking url is '{}'. Redirecting to RM app page", + originalUri == null ? "NULL" : originalUri); + ProxyUtils.sendRedirect(req, resp, + StringHelper.pjoin(rmAppPageUrlBase, id.toString())); + } else if (appReportSource == AppReportSource.AHS) { + // fallback to Application History Server app page if the application + // report was fetched from AHS + LOG.debug("Original tracking url is '{}'. Redirecting to AHS app page", + originalUri == null ? "NULL" : originalUri); + ProxyUtils.sendRedirect(req, resp, + StringHelper.pjoin(ahsAppPageUrlBase, id.toString())); + } + } else if (ProxyUriUtils.getSchemeFromUrl(originalUri).isEmpty()) { + trackingUri = + ProxyUriUtils.getUriFromAMUrl(WebAppUtils.getHttpSchemePrefix(conf), + originalUri); + } else { + trackingUri = new URI(originalUri); + } + + return trackingUri; + } + + /** + * Fetch the application report from the RM. + * + * @param id the app ID + * @return the application report + * @throws IOException if the request to the RM fails + * @throws YarnException if the request to the RM fails + */ + private FetchedAppReport getFetchedAppReport(ApplicationId id) + throws IOException, YarnException { + FetchedAppReport fetchedAppReport = getApplicationReport(id); + + if (fetchedAppReport != null) { + if ((fetchedAppReport.getAppReportSource() != AppReportSource.RM) && + (fetchedAppReport.getAppReportSource() != AppReportSource.AHS)) { + throw new UnsupportedOperationException("Application report not " + + "fetched from RM or history server."); + } + } + + return fetchedAppReport; + } + + /** + * Check whether the request is a redirect from the AM and handle it + * appropriately. This check exists to prevent the AM from forwarding back to + * the web proxy, which would contact the AM again, which would forward + * again... If this method returns true, there was a redirect, and + * it was handled by redirecting the current request to an error page. + * + * @param path the part of the request path after the app id + * @param id the app id + * @param req the request object + * @param resp the response object + * @return whether there was a redirect + * @throws IOException if a redirect fails + */ + private boolean handleRedirect(String id, HttpServletRequest req, + HttpServletResponse resp) throws IOException { + // If this isn't a redirect, we don't care. + boolean badRedirect = false; + + // If this is a redirect, check if we're calling ourselves. + try { + badRedirect = NetUtils.getLocalInetAddress(req.getRemoteHost()) != null; + } catch (SocketException ex) { + // This exception means we can't determine the calling host. Odds are + // that means it's not us. Let it go and hope it works out better next + // time. + } + + // If the proxy tries to call itself, it gets into an endless + // loop and consumes all available handler threads until the + // application completes. Redirect to the app page with a flag + // that tells it to print an appropriate error message. + if (badRedirect) { + LOG.error("The AM's web app redirected the RM web proxy's request back " + + "to the web proxy. The typical cause is that the AM is resolving " + + "the RM's address as something other than what it expects. Check " + + "your network configuration and the value of the " + + "yarn.web-proxy.address property. Once the host resolution issue " + + "has been resolved, you will likely need to delete the " + + "misbehaving application, " + id); + String redirect = StringHelper.pjoin(failurePageUrlBase, id); + LOG.error("REDIRECT: sending redirect to " + redirect); + ProxyUtils.sendRedirect(req, resp, redirect); + } + + return badRedirect; + } + /** * This method is used by Java object deserialization, to fill in the * transient {@link #trackingUriPlugins} field. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java index e7617f0e0ff..fe6fc329b58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java @@ -59,8 +59,9 @@ public class AmIpFilter implements Filter { public static final String PROXY_HOSTS_DELIMITER = ","; public static final String PROXY_URI_BASES = "PROXY_URI_BASES"; public static final String PROXY_URI_BASES_DELIMITER = ","; + private static final String PROXY_PATH = "/proxy"; //update the proxy IP list about every 5 min - private static final long updateInterval = 5 * 60 * 1000; + private static final long UPDATE_INTERVAL = 5 * 60 * 1000; private String[] proxyHosts; private Set proxyAddresses = null; @@ -96,7 +97,7 @@ public class AmIpFilter implements Filter { protected Set getProxyAddresses() throws ServletException { long now = System.currentTimeMillis(); synchronized(this) { - if(proxyAddresses == null || (lastUpdate + updateInterval) >= now) { + if (proxyAddresses == null || (lastUpdate + UPDATE_INTERVAL) >= now) { proxyAddresses = new HashSet<>(); for (String proxyHost : proxyHosts) { try { @@ -131,37 +132,52 @@ public class AmIpFilter implements Filter { HttpServletRequest httpReq = (HttpServletRequest)req; HttpServletResponse httpResp = (HttpServletResponse)resp; + if (LOG.isDebugEnabled()) { LOG.debug("Remote address for request is: {}", httpReq.getRemoteAddr()); } + if (!getProxyAddresses().contains(httpReq.getRemoteAddr())) { - String redirectUrl = findRedirectUrl(); - String target = redirectUrl + httpReq.getRequestURI(); - ProxyUtils.sendRedirect(httpReq, httpResp, target); - return; - } + StringBuilder redirect = new StringBuilder(findRedirectUrl()); - String user = null; + redirect.append(httpReq.getRequestURI()); - if (httpReq.getCookies() != null) { - for(Cookie c: httpReq.getCookies()) { - if(WebAppProxyServlet.PROXY_USER_COOKIE_NAME.equals(c.getName())){ - user = c.getValue(); - break; + int insertPoint = redirect.indexOf(PROXY_PATH); + + if (insertPoint >= 0) { + // Add /redirect as the second component of the path so that the RM web + // proxy knows that this request was a redirect. + insertPoint += PROXY_PATH.length(); + redirect.insert(insertPoint, "/redirect"); + } + + ProxyUtils.sendRedirect(httpReq, httpResp, redirect.toString()); + } else { + String user = null; + + if (httpReq.getCookies() != null) { + for(Cookie c: httpReq.getCookies()) { + if(WebAppProxyServlet.PROXY_USER_COOKIE_NAME.equals(c.getName())){ + user = c.getValue(); + break; + } } } - } - if (user == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Could not find " + WebAppProxyServlet.PROXY_USER_COOKIE_NAME - + " cookie, so user will not be set"); + if (user == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Could not find " + + WebAppProxyServlet.PROXY_USER_COOKIE_NAME + + " cookie, so user will not be set"); + } + + chain.doFilter(req, resp); + } else { + AmIpPrincipal principal = new AmIpPrincipal(user); + ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq, + principal); + + chain.doFilter(requestWrapper, resp); } - chain.doFilter(req, resp); - } else { - final AmIpPrincipal principal = new AmIpPrincipal(user); - ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq, - principal); - chain.doFilter(requestWrapper, resp); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java index 330e4de59ad..72369824d8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java @@ -155,7 +155,7 @@ public class TestWebAppProxyServlet { URL emptyUrl = new URL("http://localhost:" + proxyPort + "/proxy"); HttpURLConnection emptyProxyConn = (HttpURLConnection) emptyUrl .openConnection(); - emptyProxyConn.connect();; + emptyProxyConn.connect(); assertEquals(HttpURLConnection.HTTP_NOT_FOUND, emptyProxyConn.getResponseCode()); // wrong url. Set wrong app ID @@ -176,6 +176,25 @@ public class TestWebAppProxyServlet { assertEquals(HttpURLConnection.HTTP_OK, proxyConn.getResponseCode()); assertTrue(isResponseCookiePresent( proxyConn, "checked_application_0_0000", "true")); + + // test that redirection is squashed correctly + URL redirectUrl = new URL("http://localhost:" + proxyPort + + "/proxy/redirect/application_00_0"); + proxyConn = (HttpURLConnection) redirectUrl.openConnection(); + proxyConn.setInstanceFollowRedirects(false); + proxyConn.connect(); + assertEquals("The proxy returned an unexpected status code rather than" + + "redirecting the connection (302)", + HttpURLConnection.HTTP_MOVED_TEMP, proxyConn.getResponseCode()); + + String expected = + WebAppUtils.getResolvedRMWebAppURLWithScheme(configuration) + + "/cluster/failure/application_00_0"; + String redirect = proxyConn.getHeaderField(ProxyUtils.LOCATION); + + assertEquals("The proxy did not redirect the connection to the failure " + + "page of the RM", expected, redirect); + // cannot found application 1: null appReportFetcher.answer = 1; proxyConn = (HttpURLConnection) url.openConnection(); @@ -185,6 +204,7 @@ public class TestWebAppProxyServlet { proxyConn.getResponseCode()); assertFalse(isResponseCookiePresent( proxyConn, "checked_application_0_0000", "true")); + // cannot found application 2: ApplicationNotFoundException appReportFetcher.answer = 4; proxyConn = (HttpURLConnection) url.openConnection(); @@ -194,6 +214,7 @@ public class TestWebAppProxyServlet { proxyConn.getResponseCode()); assertFalse(isResponseCookiePresent( proxyConn, "checked_application_0_0000", "true")); + // wrong user appReportFetcher.answer = 2; proxyConn = (HttpURLConnection) url.openConnection(); @@ -203,6 +224,7 @@ public class TestWebAppProxyServlet { assertTrue(s .contains("to continue to an Application Master web interface owned by")); assertTrue(s.contains("WARNING: The following page may not be safe!")); + //case if task has a not running status appReportFetcher.answer = 3; proxyConn = (HttpURLConnection) url.openConnection(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java index 6f64777aace..9dc0ce0f10d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.webproxy.amfilter; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; +import java.net.HttpURLConnection; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -147,8 +148,8 @@ public class TestAmFilter { testFilter.init(config); HttpServletResponseForTest response = new HttpServletResponseForTest(); - // Test request should implements HttpServletRequest + // Test request should implements HttpServletRequest ServletRequest failRequest = Mockito.mock(ServletRequest.class); try { testFilter.doFilter(failRequest, response, chain); @@ -159,22 +160,32 @@ public class TestAmFilter { // request with HttpServletRequest HttpServletRequest request = Mockito.mock(HttpServletRequest.class); - Mockito.when(request.getRemoteAddr()).thenReturn("redirect"); - Mockito.when(request.getRequestURI()).thenReturn("/redirect"); + Mockito.when(request.getRemoteAddr()).thenReturn("nowhere"); + Mockito.when(request.getRequestURI()).thenReturn("/app/application_00_0"); + + // address "redirect" is not in host list for non-proxy connection testFilter.doFilter(request, response, chain); - // address "redirect" is not in host list - assertEquals(302, response.status); + assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, response.status); String redirect = response.getHeader(ProxyUtils.LOCATION); - assertEquals("http://bogus/redirect", redirect); + assertEquals("http://bogus/app/application_00_0", redirect); + + // address "redirect" is not in host list for proxy connection + Mockito.when(request.getRequestURI()).thenReturn("/proxy/application_00_0"); + testFilter.doFilter(request, response, chain); + assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, response.status); + redirect = response.getHeader(ProxyUtils.LOCATION); + assertEquals("http://bogus/proxy/redirect/application_00_0", redirect); + // "127.0.0.1" contains in host list. Without cookie Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1"); testFilter.doFilter(request, response, chain); - assertTrue(doFilterRequest .contains("javax.servlet.http.HttpServletRequest")); + // cookie added - Cookie[] cookies = new Cookie[1]; - cookies[0] = new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user"); + Cookie[] cookies = new Cookie[] { + new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user") + }; Mockito.when(request.getCookies()).thenReturn(cookies); testFilter.doFilter(request, response, chain); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 5862506755c..051509c9b51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -3237,6 +3237,7 @@ The Cluster Reservation API can be used to list reservations. When listing reser | deadline | long | The UTC time representation of the latest time within which this reservation can be allocated. | | reservation-name | string | A mnemonic name of the reservation (not a valid identifier). | | reservation-requests | object | A list of "stages" or phases of this reservation, each describing resource requirements and duration | +| priority | int | An integer representing the priority of the reservation. A lower number for priority indicates a higher priority reservation. Recurring reservations are always higher priority than non-recurring reservations. Priority for non-recurring reservations are only compared with non-recurring reservations. Likewise with recurring reservations. | ### Elements of the *reservation-requests* object @@ -3500,6 +3501,7 @@ Elements of the *reservation-definition* object | deadline | long | The UTC time representation of the latest time within which this reservation can be allocated. | | reservation-name | string | A mnemonic name of the reservation (not a valid identifier). | | reservation-requests | object | A list of "stages" or phases of this reservation, each describing resource requirements and duration | +| priority | int | An integer representing the priority of the reservation. A lower number for priority indicates a higher priority reservation. Recurring reservations are always higher priority than non-recurring reservations. Priority for non-recurring reservations are only compared with non-recurring reservations. Likewise with recurring reservations. | Elements of the *reservation-requests* object @@ -3675,6 +3677,7 @@ Elements of the *reservation-definition* object | deadline | long | The UTC time representation of the latest time within which this reservation can be allocated. | | reservation-name | string | A mnemonic name of the reservation (not a valid identifier). | | reservation-requests | object | A list of "stages" or phases of this reservation, each describing resource requirements and duration | +| priority | int | An integer representing the priority of the reservation. A lower number for priority indicates a higher priority reservation. Recurring reservations are always higher priority than non-recurring reservations. Priority for non-recurring reservations are only compared with non-recurring reservations. Likewise with recurring reservations. | Elements of the *reservation-requests* object diff --git a/pom.xml b/pom.xml index 250f5a17b13..1a3cd28a689 100644 --- a/pom.xml +++ b/pom.xml @@ -429,7 +429,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs aggregate 1024m - true true false ${maven.compile.source}