From 9e28e089c6bcb0b264751bdfa1f994ce002ef2de Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Thu, 11 Sep 2014 16:19:23 +0000 Subject: [PATCH 1/9] MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but is not documented. Contributed by Tsuyoshi OZAWA --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/resources/mapred-default.xml | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 490c0a1d4d6..0c065c56941 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -279,6 +279,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length files (jlowe) + MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but + is not documented (Tsuyoshi OZAWA via jlowe) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 703a1039569..6cefdc97b52 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -508,7 +508,12 @@ Whether to enable the small-jobs "ubertask" optimization, which runs "sufficiently small" jobs sequentially within a single JVM. "Small" is defined by the following maxmaps, maxreduces, and maxbytes - settings. Users may override this value. + settings. Note that configurations for application masters also affect + the "Small" definition - yarn.app.mapreduce.am.resource.mb must be + larger than both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb, + and yarn.app.mapreduce.am.resource.cpu-vcores must be larger than + both mapreduce.map.cpu.vcores and mapreduce.reduce.cpu.vcores to enable + ubertask. Users may override this value. From bf64fce78c5474f4d6ead839c4de18c8401a54d6 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 11 Sep 2014 10:26:13 -0700 Subject: [PATCH 2/9] HDFS-6966. Add additional unit tests for encryption zones. Contributed by Stephen Chu. --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 20 +- .../hadoop/hdfs/TestEncryptionZones.java | 187 ++++++++++++++++++ .../hdfs/TestEncryptionZonesWithHA.java | 121 ++++++++++++ 4 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index fa00d448d80..266937d88e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -635,6 +635,9 @@ Release 2.6.0 - UNRELEASED HDFS-6506. Newly moved block replica been invalidated and deleted in TestBalancer. (Binglin Chang via cnauroth) + HDFS-6966. Add additional unit tests for encryption zones. + (Stephen Chu via wang) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index e3a265fcab0..9e442ef9ba0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -1359,7 +1359,8 @@ public class DFSTestUtil { } /** - * Helper function to create a key in the Key Provider. + * Helper function to create a key in the Key Provider. Defaults + * to the first indexed NameNode's Key Provider. * * @param keyName The name of the key to create * @param cluster The cluster to create it in @@ -1368,7 +1369,22 @@ public class DFSTestUtil { public static void createKey(String keyName, MiniDFSCluster cluster, Configuration conf) throws NoSuchAlgorithmException, IOException { - KeyProvider provider = cluster.getNameNode().getNamesystem().getProvider(); + createKey(keyName, cluster, 0, conf); + } + + /** + * Helper function to create a key in the Key Provider. + * + * @param keyName The name of the key to create + * @param cluster The cluster to create it in + * @param idx The NameNode index + * @param conf Configuration to use + */ + public static void createKey(String keyName, MiniDFSCluster cluster, + int idx, Configuration conf) + throws NoSuchAlgorithmException, IOException { + NameNode nn = cluster.getNameNode(idx); + KeyProvider provider = nn.getNamesystem().getProvider(); final KeyProvider.Options options = KeyProvider.options(conf); options.setDescription(keyName); options.setBitLength(128); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index d0d29ea6d79..db3c085fe12 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -17,8 +17,14 @@ */ package org.apache.hadoop.hdfs; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.io.StringReader; +import java.io.StringWriter; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.List; @@ -52,10 +58,15 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.namenode.EncryptionFaultInjector; import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager; +import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck; +import org.apache.hadoop.hdfs.tools.DFSck; +import org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.DelegationTokenExtension; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension; import org.apache.hadoop.io.Text; @@ -78,6 +89,11 @@ import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import org.xml.sax.InputSource; +import org.xml.sax.helpers.DefaultHandler; + +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; public class TestEncryptionZones { @@ -374,6 +390,7 @@ public class TestEncryptionZones { final Path allPathFile = new Path(allPath, "file1"); final Path nonEZDir = new Path(testRoot, "nonEZDir"); final Path nonEZFile = new Path(nonEZDir, "file1"); + final Path nonexistent = new Path("/nonexistent"); final int len = 8192; fsWrapper.mkdir(testRoot, new FsPermission((short) 0777), true); @@ -421,6 +438,9 @@ public class TestEncryptionZones { assertExceptionContains("Permission denied:", e); } + assertNull("expected null for nonexistent path", + userAdmin.getEncryptionZoneForPath(nonexistent)); + // Check operation with non-ez paths assertNull("expected null for non-ez path", userAdmin.getEncryptionZoneForPath(nonEZDir)); @@ -463,9 +483,15 @@ public class TestEncryptionZones { final Path testRoot = new Path("/tmp/TestEncryptionZones"); final Path pathFoo = new Path(testRoot, "foo"); final Path pathFooBaz = new Path(pathFoo, "baz"); + final Path pathFooBazFile = new Path(pathFooBaz, "file"); + final Path pathFooBar = new Path(pathFoo, "bar"); + final Path pathFooBarFile = new Path(pathFooBar, "file"); + final int len = 8192; wrapper.mkdir(pathFoo, FsPermission.getDirDefault(), true); dfsAdmin.createEncryptionZone(pathFoo, TEST_KEY); wrapper.mkdir(pathFooBaz, FsPermission.getDirDefault(), true); + DFSTestUtil.createFile(fs, pathFooBazFile, len, (short) 1, 0xFEED); + String contents = DFSTestUtil.readFile(fs, pathFooBazFile); try { wrapper.rename(pathFooBaz, testRoot); } catch (IOException e) { @@ -473,6 +499,13 @@ public class TestEncryptionZones { " an encryption zone.", e ); } + + // Verify that we can rename dir and files within an encryption zone. + assertTrue(fs.rename(pathFooBaz, pathFooBar)); + assertTrue("Rename of dir and file within ez failed", + !wrapper.exists(pathFooBaz) && wrapper.exists(pathFooBar)); + assertEquals("Renamed file contents not the same", + contents, DFSTestUtil.readFile(fs, pathFooBarFile)); } @Test(timeout = 60000) @@ -806,4 +839,158 @@ public class TestEncryptionZones { Assert.assertEquals(tokens[1], testToken); Assert.assertEquals(1, creds.numberOfTokens()); } + + /** + * Test running fsck on a system with encryption zones. + */ + @Test(timeout = 60000) + public void testFsckOnEncryptionZones() throws Exception { + final int len = 8196; + final Path zoneParent = new Path("/zones"); + final Path zone1 = new Path(zoneParent, "zone1"); + final Path zone1File = new Path(zone1, "file"); + fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true); + dfsAdmin.createEncryptionZone(zone1, TEST_KEY); + DFSTestUtil.createFile(fs, zone1File, len, (short) 1, 0xFEED); + ByteArrayOutputStream bStream = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bStream, true); + int errCode = ToolRunner.run(new DFSck(conf, out), + new String[]{ "/" }); + assertEquals("Fsck ran with non-zero error code", 0, errCode); + String result = bStream.toString(); + assertTrue("Fsck did not return HEALTHY status", + result.contains(NamenodeFsck.HEALTHY_STATUS)); + + // Run fsck directly on the encryption zone instead of root + errCode = ToolRunner.run(new DFSck(conf, out), + new String[]{ zoneParent.toString() }); + assertEquals("Fsck ran with non-zero error code", 0, errCode); + result = bStream.toString(); + assertTrue("Fsck did not return HEALTHY status", + result.contains(NamenodeFsck.HEALTHY_STATUS)); + } + + /** + * Test correctness of successive snapshot creation and deletion + * on a system with encryption zones. + */ + @Test(timeout = 60000) + public void testSnapshotsOnEncryptionZones() throws Exception { + final int len = 8196; + final Path zoneParent = new Path("/zones"); + final Path zone = new Path(zoneParent, "zone"); + final Path zoneFile = new Path(zone, "zoneFile"); + fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true); + dfsAdmin.allowSnapshot(zoneParent); + dfsAdmin.createEncryptionZone(zone, TEST_KEY); + DFSTestUtil.createFile(fs, zoneFile, len, (short) 1, 0xFEED); + String contents = DFSTestUtil.readFile(fs, zoneFile); + final Path snap1 = fs.createSnapshot(zoneParent); + final Path snap1Zone = new Path(snap1, zone.getName()); + assertEquals("Got unexpected ez path", zone.toString(), + dfsAdmin.getEncryptionZoneForPath(snap1Zone).getPath().toString()); + + // Now delete the encryption zone, recreate the dir, and take another snapshot + fsWrapper.delete(zone, true); + fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true); + final Path snap2 = fs.createSnapshot(zoneParent); + final Path snap2Zone = new Path(snap2, zone.getName()); + assertNull("Expected null ez path", + dfsAdmin.getEncryptionZoneForPath(snap2Zone)); + + // Create the encryption zone again + dfsAdmin.createEncryptionZone(zone, TEST_KEY); + final Path snap3 = fs.createSnapshot(zoneParent); + final Path snap3Zone = new Path(snap3, zone.getName()); + assertEquals("Got unexpected ez path", zone.toString(), + dfsAdmin.getEncryptionZoneForPath(snap3Zone).getPath().toString()); + + // Verify contents of the snapshotted file + final Path snapshottedZoneFile = new Path( + snap1.toString() + "/" + zone.getName() + "/" + zoneFile.getName()); + assertEquals("Contents of snapshotted file have changed unexpectedly", + contents, DFSTestUtil.readFile(fs, snapshottedZoneFile)); + + // Now delete the snapshots out of order and verify the zones are still correct + fs.deleteSnapshot(zoneParent, snap2.getName()); + assertEquals("Got unexpected ez path", zone.toString(), + dfsAdmin.getEncryptionZoneForPath(snap1Zone).getPath().toString()); + assertEquals("Got unexpected ez path", zone.toString(), + dfsAdmin.getEncryptionZoneForPath(snap3Zone).getPath().toString()); + fs.deleteSnapshot(zoneParent, snap1.getName()); + assertEquals("Got unexpected ez path", zone.toString(), + dfsAdmin.getEncryptionZoneForPath(snap3Zone).getPath().toString()); + } + + /** + * Verify symlinks can be created in encryption zones and that + * they function properly when the target is in the same + * or different ez. + */ + @Test(timeout = 60000) + public void testEncryptionZonesWithSymlinks() throws Exception { + // Verify we can create an encryption zone over both link and target + final int len = 8192; + final Path parent = new Path("/parent"); + final Path linkParent = new Path(parent, "symdir1"); + final Path targetParent = new Path(parent, "symdir2"); + final Path link = new Path(linkParent, "link"); + final Path target = new Path(targetParent, "target"); + fs.mkdirs(parent); + dfsAdmin.createEncryptionZone(parent, TEST_KEY); + fs.mkdirs(linkParent); + fs.mkdirs(targetParent); + DFSTestUtil.createFile(fs, target, len, (short)1, 0xFEED); + String content = DFSTestUtil.readFile(fs, target); + fs.createSymlink(target, link, false); + assertEquals("Contents read from link are not the same as target", + content, DFSTestUtil.readFile(fs, link)); + fs.delete(parent, true); + + // Now let's test when the symlink and target are in different + // encryption zones + fs.mkdirs(linkParent); + fs.mkdirs(targetParent); + dfsAdmin.createEncryptionZone(linkParent, TEST_KEY); + dfsAdmin.createEncryptionZone(targetParent, TEST_KEY); + DFSTestUtil.createFile(fs, target, len, (short)1, 0xFEED); + content = DFSTestUtil.readFile(fs, target); + fs.createSymlink(target, link, false); + assertEquals("Contents read from link are not the same as target", + content, DFSTestUtil.readFile(fs, link)); + fs.delete(link, true); + fs.delete(target, true); + } + + /** + * Test running the OfflineImageViewer on a system with encryption zones. + */ + @Test(timeout = 60000) + public void testOfflineImageViewerOnEncryptionZones() throws Exception { + final int len = 8196; + final Path zoneParent = new Path("/zones"); + final Path zone1 = new Path(zoneParent, "zone1"); + final Path zone1File = new Path(zone1, "file"); + fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true); + dfsAdmin.createEncryptionZone(zone1, TEST_KEY); + DFSTestUtil.createFile(fs, zone1File, len, (short) 1, 0xFEED); + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false); + fs.saveNamespace(); + + File originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil + .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0)); + if (originalFsimage == null) { + throw new RuntimeException("Didn't generate or can't find fsimage"); + } + + // Run the XML OIV processor + StringWriter output = new StringWriter(); + PrintWriter pw = new PrintWriter(output); + PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), pw); + v.visit(new RandomAccessFile(originalFsimage, "r")); + final String xml = output.getBuffer().toString(); + SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); + parser.parse(new InputSource(new StringReader(xml)), new DefaultHandler()); + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java new file mode 100644 index 00000000000..b6040045ac4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; +import org.apache.hadoop.crypto.key.KeyProviderFactory; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.FileSystemTestHelper; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; +import org.apache.hadoop.hdfs.client.HdfsAdmin; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +/** + * Tests interaction of encryption zones with HA failover. + */ +public class TestEncryptionZonesWithHA { + private Configuration conf; + private MiniDFSCluster cluster; + private NameNode nn0; + private NameNode nn1; + private DistributedFileSystem fs; + private HdfsAdmin dfsAdmin0; + private HdfsAdmin dfsAdmin1; + private FileSystemTestHelper fsHelper; + private File testRootDir; + + private final String TEST_KEY = "testKey"; + + + @Before + public void setupCluster() throws Exception { + conf = new Configuration(); + conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); + HAUtil.setAllowStandbyReads(conf, true); + fsHelper = new FileSystemTestHelper(); + String testRoot = fsHelper.getTestRootDir(); + testRootDir = new File(testRoot).getAbsoluteFile(); + conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, + JavaKeyStoreProvider.SCHEME_NAME + "://file" + testRootDir + "/test.jks" + ); + + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(MiniDFSNNTopology.simpleHATopology()) + .numDataNodes(1) + .build(); + cluster.waitActive(); + cluster.transitionToActive(0); + + fs = (DistributedFileSystem)HATestUtil.configureFailoverFs(cluster, conf); + DFSTestUtil.createKey(TEST_KEY, cluster, 0, conf); + DFSTestUtil.createKey(TEST_KEY, cluster, 1, conf); + nn0 = cluster.getNameNode(0); + nn1 = cluster.getNameNode(1); + dfsAdmin0 = new HdfsAdmin(cluster.getURI(0), conf); + dfsAdmin1 = new HdfsAdmin(cluster.getURI(1), conf); + KeyProviderCryptoExtension nn0Provider = + cluster.getNameNode(0).getNamesystem().getProvider(); + fs.getClient().provider = nn0Provider; + } + + @After + public void shutdownCluster() throws IOException { + if (cluster != null) { + cluster.shutdown(); + } + } + + /** + * Test that encryption zones are properly tracked by the standby. + */ + @Test(timeout = 60000) + public void testEncryptionZonesTrackedOnStandby() throws Exception { + final int len = 8196; + final Path dir = new Path("/enc"); + final Path dirChild = new Path(dir, "child"); + final Path dirFile = new Path(dir, "file"); + fs.mkdir(dir, FsPermission.getDirDefault()); + dfsAdmin0.createEncryptionZone(dir, TEST_KEY); + fs.mkdir(dirChild, FsPermission.getDirDefault()); + DFSTestUtil.createFile(fs, dirFile, len, (short) 1, 0xFEED); + String contents = DFSTestUtil.readFile(fs, dirFile); + + // Failover the current standby to active. + HATestUtil.waitForStandbyToCatchUp(nn0, nn1); + cluster.shutdownNameNode(0); + cluster.transitionToActive(1); + + Assert.assertEquals("Got unexpected ez path", dir.toString(), + dfsAdmin1.getEncryptionZoneForPath(dir).getPath().toString()); + Assert.assertEquals("Got unexpected ez path", dir.toString(), + dfsAdmin1.getEncryptionZoneForPath(dirChild).getPath().toString()); + Assert.assertEquals("File contents after failover were changed", + contents, DFSTestUtil.readFile(fs, dirFile)); + } + +} From d1dab6cc63abf5d44533ad9b39e32c582327627b Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 11 Sep 2014 10:32:20 -0700 Subject: [PATCH 3/9] HDFS-6621. Hadoop Balancer prematurely exits iterations. Contributed by Rafal Wodjdyla and Benjamin Bowman. --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/server/balancer/Dispatcher.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 266937d88e3..cfa9344f3c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -638,6 +638,9 @@ Release 2.6.0 - UNRELEASED HDFS-6966. Add additional unit tests for encryption zones. (Stephen Chu via wang) + HDFS-6621. Hadoop Balancer prematurely exits iterations. + (Rafal Wojdyla and Benjamin Bowman via wang) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java index b7dceb6f48d..14d598936ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java @@ -654,6 +654,8 @@ public class Dispatcher { && (!srcBlocks.isEmpty() || blocksToReceive > 0)) { final PendingMove p = chooseNextMove(); if (p != null) { + // Reset no pending move counter + noPendingMoveIteration=0; // move the block moveExecutor.execute(new Runnable() { @Override From 1e684995d7ac20d1c27ca1c1bef37dd2ee00e630 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 11 Sep 2014 10:59:09 -0700 Subject: [PATCH 4/9] HDFS-7045. Fix NameNode deadlock when opening file under /.reserved path. Contributed by Yi Liu. --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../java/org/apache/hadoop/hdfs/TestRead.java | 22 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index cfa9344f3c0..4c9dbafe054 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -641,6 +641,9 @@ Release 2.6.0 - UNRELEASED HDFS-6621. Hadoop Balancer prematurely exits iterations. (Rafal Wojdyla and Benjamin Bowman via wang) + HDFS-7045. Fix NameNode deadlock when opening file under /.reserved path. + (Yi Liu via wang) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index c1744f6421b..0ebbc1de797 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1821,8 +1821,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, checkOperation(OperationCategory.WRITE); writeLock(); // writelock is needed to set accesstime } - src = resolvePath(src, pathComponents); try { + src = resolvePath(src, pathComponents); if (isReadOp) { checkOperation(OperationCategory.READ); } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java index c8d50297b3d..378d8bba153 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs; +import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; @@ -80,4 +81,25 @@ public class TestRead { testEOF(cluster, 10000); cluster.shutdown(); } + + /** + * Regression test for HDFS-7045. + * If deadlock happen, the test will time out. + * @throws Exception + */ + @Test(timeout=60000) + public void testReadReservedPath() throws Exception { + Configuration conf = new Configuration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf). + numDataNodes(1).format(true).build(); + try { + FileSystem fs = cluster.getFileSystem(); + fs.open(new Path("/.reserved/.inodes/file")); + Assert.fail("Open a non existing file should fail."); + } catch (FileNotFoundException e) { + // Expected + } finally { + cluster.shutdown(); + } + } } From a123d02f4190ceadf94a66bc37a89c7f4e2ede51 Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Thu, 11 Sep 2014 11:31:58 -0700 Subject: [PATCH 5/9] HADOOP-9540: move CHANGES.txt entry to 2.6 --- hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3cea14a317c..2ae642cc8a8 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -102,9 +102,6 @@ Trunk (Unreleased) HADOOP-9486. Promoted Windows and Shell related utils from YARN to Hadoop Common. (Chris Nauroth via vinodkv) - HADOOP-9540. Expose the InMemoryS3 and S3N FilesystemStores implementations - for Unit testing. (Hari via stevel) - HADOOP-8844. Add a plaintext fs -text test-case. (Akira AJISAKA via harsh) @@ -511,6 +508,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10758. KMS: add ACLs on per key basis. (tucu) + HADOOP-9540. Expose the InMemoryS3 and S3N FilesystemStores implementations + for Unit testing. (Hari via stevel) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) From ac35e3464ff1b5bacf416fb78112681f0a9e7ed8 Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Thu, 11 Sep 2014 11:46:35 -0700 Subject: [PATCH 6/9] HADOOP-10373: move CHANGES.txt entry to branch-2.6 --- hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2ae642cc8a8..ffb8ede8188 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -126,8 +126,6 @@ Trunk (Unreleased) HADOOP-11041. VersionInfo specifies subversion (Tsuyoshi OZAWA via aw) - HADOOP-10373 create tools/hadoop-amazon for aws/EMR support (stevel) - BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. @@ -511,6 +509,8 @@ Release 2.6.0 - UNRELEASED HADOOP-9540. Expose the InMemoryS3 and S3N FilesystemStores implementations for Unit testing. (Hari via stevel) + HADOOP-10373 create tools/hadoop-amazon for aws/EMR support (stevel) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) From c11ada5ea6d17321626e5a9a4152ff857d03aee2 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Thu, 11 Sep 2014 12:06:06 -0700 Subject: [PATCH 7/9] YARN-2534. FairScheduler: Potential integer overflow calculating totalMaxShare. (Zhihai Xu via kasha) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../fair/policies/ComputeFairShares.java | 7 ++- .../scheduler/fair/TestFairScheduler.java | 48 +++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6a871a58a8b..eac9f3c73c4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -323,6 +323,9 @@ Release 2.6.0 - UNRELEASED YARN-2158. Fixed TestRMWebServicesAppsModification#testSingleAppKill test failure. (Varun Vasudev via jianhe) + YARN-2534. FairScheduler: Potential integer overflow calculating totalMaxShare. + (Zhihai Xu via kasha) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java index 12ddde2ed3e..3bea985edbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java @@ -124,11 +124,10 @@ public class ComputeFairShares { int totalMaxShare = 0; for (Schedulable sched : schedulables) { int maxShare = getResourceValue(sched.getMaxShare(), type); - if (maxShare == Integer.MAX_VALUE) { - totalMaxShare = Integer.MAX_VALUE; + totalMaxShare = (int) Math.min((long)maxShare + (long)totalMaxShare, + Integer.MAX_VALUE); + if (totalMaxShare == Integer.MAX_VALUE) { break; - } else { - totalMaxShare += maxShare; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 3a0dac3b424..1c9b2894fc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -308,6 +308,54 @@ public class TestFairScheduler extends FairSchedulerTestBase { } } + @Test + public void testFairShareWithMaxResources() throws IOException { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + // set queueA and queueB maxResources, + // the sum of queueA and queueB maxResources is more than + // Integer.MAX_VALUE. + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("1073741824 mb 1000 vcores"); + out.println(".25"); + out.println(""); + out.println(""); + out.println("1073741824 mb 1000 vcores"); + out.println(".75"); + out.println(""); + out.println(""); + out.close(); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + // Add one big node (only care about aggregate capacity) + RMNode node1 = + MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1, + "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + // Queue A wants 1 * 1024. + createSchedulingRequest(1 * 1024, "queueA", "user1"); + // Queue B wants 6 * 1024 + createSchedulingRequest(6 * 1024, "queueB", "user1"); + + scheduler.update(); + + FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue( + "queueA", false); + // queueA's weight is 0.25, so its fair share should be 2 * 1024. + assertEquals(2 * 1024, queue.getFairShare().getMemory()); + // queueB's weight is 0.75, so its fair share should be 6 * 1024. + queue = scheduler.getQueueManager().getLeafQueue( + "queueB", false); + assertEquals(6 * 1024, queue.getFairShare().getMemory()); + } + @Test public void testFairShareWithZeroWeight() throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); From 80ac6aabcea9f808fd55504cdaef2da7b50da7f1 Mon Sep 17 00:00:00 2001 From: cnauroth Date: Thu, 11 Sep 2014 12:16:19 -0700 Subject: [PATCH 8/9] HDFS-7042. Upgrade fails for Windows HA cluster due to file locks held during rename in JournalNode. Contributed by Chris Nauroth. --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/qjournal/server/Journal.java | 8 +++++++- .../org/apache/hadoop/hdfs/util/BestEffortLongFile.java | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 4c9dbafe054..a11371aa3a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -751,6 +751,9 @@ Release 2.6.0 - UNRELEASED HDFS-6776. Using distcp to copy data between insecure and secure cluster via webdhfs doesn't work. (yzhangal via tucu) + HDFS-7042. Upgrade fails for Windows HA cluster due to file locks held during + rename in JournalNode. (cnauroth) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java index b36e547056e..c79e269b178 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java @@ -988,6 +988,9 @@ public class Journal implements Closeable { } public synchronized void doPreUpgrade() throws IOException { + // Do not hold file lock on committedTxnId, because the containing + // directory will be renamed. It will be reopened lazily on next access. + committedTxnId.close(); storage.getJournalManager().doPreUpgrade(); } @@ -1037,7 +1040,10 @@ public class Journal implements Closeable { targetLayoutVersion); } - public void doRollback() throws IOException { + public synchronized void doRollback() throws IOException { + // Do not hold file lock on committedTxnId, because the containing + // directory will be renamed. It will be reopened lazily on next access. + committedTxnId.close(); storage.getJournalManager().doRollback(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java index b2626e9d181..2fdf5472bee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java @@ -112,6 +112,7 @@ public class BestEffortLongFile implements Closeable { public void close() throws IOException { if (ch != null) { ch.close(); + ch = null; } } } From 581176cdc868b8701a761b5a0d52f771cbba709a Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Thu, 11 Sep 2014 19:53:25 +0000 Subject: [PATCH 9/9] HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when value is NaN. Contributed by Mit Desai --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/util/Progress.java | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ffb8ede8188..3bf92ec500e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -784,6 +784,9 @@ Release 2.6.0 - UNRELEASED HADOOP-9989. Bug introduced in HADOOP-9374, which parses the -tokenCacheFile as binary file but set it to the configuration as JSON file. (zxu via tucu) + HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when + value is NaN (Mit Desai via jlowe) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java index e3445b5da6b..3fbc9350f17 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java @@ -162,27 +162,27 @@ public class Progress { public synchronized void set(float progress) { if (Float.isNaN(progress)) { progress = 0; - LOG.warn("Illegal progress value found, progress is Float.NaN. " + + LOG.debug("Illegal progress value found, progress is Float.NaN. " + "Progress will be changed to 0"); } else if (progress == Float.NEGATIVE_INFINITY) { progress = 0; - LOG.warn("Illegal progress value found, progress is " + + LOG.debug("Illegal progress value found, progress is " + "Float.NEGATIVE_INFINITY. Progress will be changed to 0"); } else if (progress < 0) { progress = 0; - LOG.warn("Illegal progress value found, progress is less than 0." + + LOG.debug("Illegal progress value found, progress is less than 0." + " Progress will be changed to 0"); } else if (progress > 1) { progress = 1; - LOG.warn("Illegal progress value found, progress is larger than 1." + + LOG.debug("Illegal progress value found, progress is larger than 1." + " Progress will be changed to 1"); } else if (progress == Float.POSITIVE_INFINITY) { progress = 1; - LOG.warn("Illegal progress value found, progress is " + + LOG.debug("Illegal progress value found, progress is " + "Float.POSITIVE_INFINITY. Progress will be changed to 1"); } this.progress = progress;