diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 5b513e73f15..b9dbc73a712 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1880,6 +1880,17 @@ public class YarnConfiguration extends Configuration { */ public static final String NM_MIN_PER_DISK_FREE_SPACE_MB = NM_DISK_HEALTH_CHECK_PREFIX + "min-free-space-per-disk-mb"; + /** + * The minimum space that must be available on an offline + * disk for it to be marked as online. The value should not be less + * than NM_MIN_PER_DISK_FREE_SPACE_MB. If its value is less than + * NM_MIN_PER_DISK_FREE_SPACE_MB or is not set, it will be set to the + * same value as NM_MIN_PER_DISK_FREE_SPACE_MB. + * This applies to nm-local-dirs and nm-log-dirs. + */ + public static final String NM_WM_HIGH_PER_DISK_FREE_SPACE_MB = + NM_DISK_HEALTH_CHECK_PREFIX + + "min-free-space-per-disk-watermark-high-mb"; /** * By default, all of the disk can be used before it is marked as offline. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 1651458e5b1..e5e8ed9f352 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1636,13 +1636,26 @@ - The minimum space that must be available on a disk for - it to be used. This applies to yarn.nodemanager.local-dirs and + The minimum space in megabytes that must be available on a disk for + it to be used. If space on a disk falls below this threshold, it will be marked + as bad. This applies to yarn.nodemanager.local-dirs and yarn.nodemanager.log-dirs. yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb 0 + + The minimum space in megabytes that must be available on a bad + disk for it to be marked as good. This value should not be less + than yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb. + If it is less than yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb, + or it is not set, it will be set to the + same value as yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb. + This applies to yarn.nodemanager.local-dirs and yarn.nodemanager.log-dirs. + yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb + 0 + + The path to the Linux container executor. yarn.nodemanager.linux-container-executor.path diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java index 3462aa47ebe..ded1ed1fdcf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java @@ -110,7 +110,8 @@ public class DirectoryCollection { private float diskUtilizationPercentageCutoffHigh; private float diskUtilizationPercentageCutoffLow; - private long diskUtilizationSpaceCutoff; + private long diskFreeSpaceCutoffLow; + private long diskFreeSpaceCutoffHigh; private int goodDirsDiskUtilizationPercentage; @@ -123,7 +124,7 @@ public class DirectoryCollection { * directories to be monitored */ public DirectoryCollection(String[] dirs) { - this(dirs, 100.0F, 100.0F, 0); + this(dirs, 100.0F, 100.0F, 0, 0); } /** @@ -139,7 +140,7 @@ public class DirectoryCollection { * */ public DirectoryCollection(String[] dirs, float utilizationPercentageCutOff) { - this(dirs, utilizationPercentageCutOff, utilizationPercentageCutOff, 0); + this(dirs, utilizationPercentageCutOff, utilizationPercentageCutOff, 0, 0); } /** @@ -154,7 +155,26 @@ public class DirectoryCollection { * */ public DirectoryCollection(String[] dirs, long utilizationSpaceCutOff) { - this(dirs, 100.0F, 100.0F, utilizationSpaceCutOff); + this(dirs, 100.0F, 100.0F, utilizationSpaceCutOff, utilizationSpaceCutOff); + } + + /** + * Create collection for the directories specified. Users must specify the + * minimum amount of free space that must be available for the dir to be used. + * + * @param dirs + * directories to be monitored + * @param utilizationSpaceCutOffLow + * minimum space, in MB, that must be available on the disk for the + * dir to be taken out of the good dirs list + * @param utilizationSpaceCutOffHigh + * minimum space, in MB, that must be available on the disk for the + * dir to be moved from the bad dirs list to the good dirs list + */ + public DirectoryCollection(String[] dirs, long utilizationSpaceCutOffLow, + long utilizationSpaceCutOffHigh) { + this(dirs, 100.0F, 100.0F, utilizationSpaceCutOffLow, + utilizationSpaceCutOffHigh); } /** @@ -162,7 +182,7 @@ public class DirectoryCollection { * maximum percentage of disk utilization allowed and the minimum amount of * free space that must be available for the dir to be used. If either check * fails the dir is removed from the good dirs list. - * + * * @param dirs * directories to be monitored * @param utilizationPercentageCutOffHigh @@ -174,12 +194,41 @@ public class DirectoryCollection { * @param utilizationSpaceCutOff * minimum space, in MB, that must be available on the disk for the * dir to be marked as good - * + */ + public DirectoryCollection(String[] dirs, + float utilizationPercentageCutOffHigh, + float utilizationPercentageCutOffLow, long utilizationSpaceCutOff) { + this(dirs, utilizationPercentageCutOffHigh, + utilizationPercentageCutOffLow, utilizationSpaceCutOff, + utilizationSpaceCutOff); + } + + /** + * Create collection for the directories specified. Users must specify the + * maximum percentage of disk utilization allowed and the minimum amount of + * free space that must be available for the dir to be used. If either check + * fails the dir is removed from the good dirs list. + * + * @param dirs + * directories to be monitored + * @param utilizationPercentageCutOffHigh + * percentage of disk that can be used before the dir is taken out + * of the good dirs list + * @param utilizationPercentageCutOffLow + * percentage of disk that can be used when the dir is moved from + * the bad dirs list to the good dirs list + * @param utilizationSpaceCutOffLow + * minimum space, in MB, that must be available on the disk for the + * dir to be taken out of the good dirs list + * @param utilizationSpaceCutOffHigh + * minimum space, in MB, that must be available on the disk for the + * dir to be moved from the bad dirs list to the good dirs list */ public DirectoryCollection(String[] dirs, float utilizationPercentageCutOffHigh, float utilizationPercentageCutOffLow, - long utilizationSpaceCutOff) { + long utilizationSpaceCutOffLow, + long utilizationSpaceCutOffHigh) { conf = new YarnConfiguration(); try { String diskValidatorName = conf.get(YarnConfiguration.DISK_VALIDATOR, @@ -199,12 +248,10 @@ public class DirectoryCollection { this.readLock = lock.readLock(); this.writeLock = lock.writeLock(); - diskUtilizationPercentageCutoffHigh = Math.max(0.0F, Math.min(100.0F, - utilizationPercentageCutOffHigh)); - diskUtilizationPercentageCutoffLow = Math.max(0.0F, Math.min( - diskUtilizationPercentageCutoffHigh, utilizationPercentageCutOffLow)); - diskUtilizationSpaceCutoff = - utilizationSpaceCutOff < 0 ? 0 : utilizationSpaceCutOff; + setDiskUtilizationPercentageCutoff(utilizationPercentageCutOffHigh, + utilizationPercentageCutOffLow); + setDiskUtilizationSpaceCutoff(utilizationSpaceCutOffLow, + utilizationSpaceCutOffHigh); dirsChangeListeners = Collections.newSetFromMap( new ConcurrentHashMap()); @@ -471,6 +518,8 @@ public class DirectoryCollection { diskValidator.checkStatus(testDir); float diskUtilizationPercentageCutoff = goodDirs.contains(dir) ? diskUtilizationPercentageCutoffHigh : diskUtilizationPercentageCutoffLow; + long diskFreeSpaceCutoff = goodDirs.contains(dir) ? + diskFreeSpaceCutoffLow : diskFreeSpaceCutoffHigh; if (isDiskUsageOverPercentageLimit(testDir, diskUtilizationPercentageCutoff)) { msg = @@ -480,9 +529,9 @@ public class DirectoryCollection { ret.put(dir, new DiskErrorInformation(DiskErrorCause.DISK_FULL, msg)); continue; - } else if (isDiskFreeSpaceUnderLimit(testDir)) { + } else if (isDiskFreeSpaceUnderLimit(testDir, diskFreeSpaceCutoff)) { msg = - "free space below limit of " + diskUtilizationSpaceCutoff + "free space below limit of " + diskFreeSpaceCutoff + "MB"; ret.put(dir, new DiskErrorInformation(DiskErrorCause.DISK_FULL, msg)); @@ -505,9 +554,10 @@ public class DirectoryCollection { || usedPercentage >= 100.0F); } - private boolean isDiskFreeSpaceUnderLimit(File dir) { + private boolean isDiskFreeSpaceUnderLimit(File dir, + long freeSpaceCutoff) { long freeSpace = dir.getUsableSpace() / (1024 * 1024); - return freeSpace < this.diskUtilizationSpaceCutoff; + return freeSpace < freeSpaceCutoff; } private void createDir(FileContext localFs, Path dir, FsPermission perm) @@ -550,13 +600,29 @@ public class DirectoryCollection { } public long getDiskUtilizationSpaceCutoff() { - return diskUtilizationSpaceCutoff; + return getDiskUtilizationSpaceCutoffLow(); } - public void setDiskUtilizationSpaceCutoff(long diskUtilizationSpaceCutoff) { - diskUtilizationSpaceCutoff = - diskUtilizationSpaceCutoff < 0 ? 0 : diskUtilizationSpaceCutoff; - this.diskUtilizationSpaceCutoff = diskUtilizationSpaceCutoff; + @VisibleForTesting + long getDiskUtilizationSpaceCutoffLow() { + return diskFreeSpaceCutoffLow; + } + + @VisibleForTesting + long getDiskUtilizationSpaceCutoffHigh() { + return diskFreeSpaceCutoffHigh; + } + + public void setDiskUtilizationSpaceCutoff(long freeSpaceCutoff) { + setDiskUtilizationSpaceCutoff(freeSpaceCutoff, + freeSpaceCutoff); + } + + public void setDiskUtilizationSpaceCutoff(long freeSpaceCutoffLow, + long freeSpaceCutoffHigh) { + diskFreeSpaceCutoffLow = Math.max(0, freeSpaceCutoffLow); + diskFreeSpaceCutoffHigh = Math.max(diskFreeSpaceCutoffLow, + Math.max(0, freeSpaceCutoffHigh)); } private void setGoodDirsDiskUtilizationPercentage() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java index 6eabd0dd5be..ac0b84d1b95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java @@ -137,23 +137,38 @@ public class LocalDirsHandlerService extends AbstractService { " is not configured properly."); lowUsableSpacePercentagePerDisk = highUsableSpacePercentagePerDisk; } - long minFreeSpacePerDiskMB = + long lowMinFreeSpacePerDiskMB = conf.getLong(YarnConfiguration.NM_MIN_PER_DISK_FREE_SPACE_MB, - YarnConfiguration.DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB); + YarnConfiguration.DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB); + long highMinFreeSpacePerDiskMB = + conf.getLong(YarnConfiguration.NM_WM_HIGH_PER_DISK_FREE_SPACE_MB, + lowMinFreeSpacePerDiskMB); + if (highMinFreeSpacePerDiskMB < lowMinFreeSpacePerDiskMB) { + LOG.warn("Using " + YarnConfiguration. + NM_MIN_PER_DISK_FREE_SPACE_MB + " as " + + YarnConfiguration.NM_WM_HIGH_PER_DISK_FREE_SPACE_MB + + ", because " + YarnConfiguration. + NM_WM_HIGH_PER_DISK_FREE_SPACE_MB + + " is not configured properly."); + highMinFreeSpacePerDiskMB = lowMinFreeSpacePerDiskMB; + } + localDirs = new DirectoryCollection( validatePaths(conf .getTrimmedStrings(YarnConfiguration.NM_LOCAL_DIRS)), highUsableSpacePercentagePerDisk, lowUsableSpacePercentagePerDisk, - minFreeSpacePerDiskMB); + lowMinFreeSpacePerDiskMB, + highMinFreeSpacePerDiskMB); logDirs = new DirectoryCollection( validatePaths(conf .getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)), highUsableSpacePercentagePerDisk, lowUsableSpacePercentagePerDisk, - minFreeSpacePerDiskMB); + lowMinFreeSpacePerDiskMB, + highMinFreeSpacePerDiskMB); String local = conf.get(YarnConfiguration.NM_LOCAL_DIRS); conf.set(NM_GOOD_LOCAL_DIRS, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java index 095f21a4f4b..b99c7d625ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java @@ -203,12 +203,30 @@ public class TestDirectoryCollection { Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffLow(), delta); - long spaceValue = 57; - dc.setDiskUtilizationSpaceCutoff(spaceValue); - Assert.assertEquals(spaceValue, dc.getDiskUtilizationSpaceCutoff()); - spaceValue = -57; - dc.setDiskUtilizationSpaceCutoff(spaceValue); - Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoff()); + long lowSpaceValue = 57; + dc.setDiskUtilizationSpaceCutoff(lowSpaceValue); + Assert.assertEquals(lowSpaceValue, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(lowSpaceValue, dc.getDiskUtilizationSpaceCutoffHigh()); + long highSpaceValue = 73; + dc.setDiskUtilizationSpaceCutoff(lowSpaceValue, highSpaceValue); + Assert.assertEquals(lowSpaceValue, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(highSpaceValue, dc.getDiskUtilizationSpaceCutoffHigh()); + lowSpaceValue = -57; + dc.setDiskUtilizationSpaceCutoff(lowSpaceValue); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffHigh()); + dc.setDiskUtilizationSpaceCutoff(lowSpaceValue, highSpaceValue); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(highSpaceValue, dc.getDiskUtilizationSpaceCutoffHigh()); + highSpaceValue = -10; + dc.setDiskUtilizationSpaceCutoff(lowSpaceValue, highSpaceValue); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffHigh()); + lowSpaceValue = 33; + dc.setDiskUtilizationSpaceCutoff(lowSpaceValue, highSpaceValue); + Assert.assertEquals(lowSpaceValue, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(lowSpaceValue, dc.getDiskUtilizationSpaceCutoffHigh()); + } @Test @@ -273,42 +291,96 @@ public class TestDirectoryCollection { delta); Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffLow(), delta); - Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoff()); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffHigh()); dc = new DirectoryCollection(dirs, 57.5F); Assert.assertEquals(57.5F, dc.getDiskUtilizationPercentageCutoffHigh(), delta); Assert.assertEquals(57.5F, dc.getDiskUtilizationPercentageCutoffLow(), delta); - Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoff()); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffHigh()); dc = new DirectoryCollection(dirs, 57); Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffHigh(), delta); Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffLow(), delta); - Assert.assertEquals(57, dc.getDiskUtilizationSpaceCutoff()); + Assert.assertEquals(57, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(57, dc.getDiskUtilizationSpaceCutoffHigh()); + + dc = new DirectoryCollection(dirs, 57, 73); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffHigh(), + delta); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffLow(), + delta); + Assert.assertEquals(57, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(73, dc.getDiskUtilizationSpaceCutoffHigh()); + + dc = new DirectoryCollection(dirs, 57, 33); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffHigh(), + delta); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffLow(), + delta); + Assert.assertEquals(57, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(57, dc.getDiskUtilizationSpaceCutoffHigh()); + + dc = new DirectoryCollection(dirs, 57, -33); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffHigh(), + delta); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffLow(), + delta); + Assert.assertEquals(57, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(57, dc.getDiskUtilizationSpaceCutoffHigh()); + + dc = new DirectoryCollection(dirs, -57, -33); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffHigh(), + delta); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffLow(), + delta); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffHigh()); + + dc = new DirectoryCollection(dirs, -57, 33); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffHigh(), + delta); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffLow(), + delta); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(33, dc.getDiskUtilizationSpaceCutoffHigh()); dc = new DirectoryCollection(dirs, 57.5F, 50.5F, 67); Assert.assertEquals(57.5F, dc.getDiskUtilizationPercentageCutoffHigh(), delta); Assert.assertEquals(50.5F, dc.getDiskUtilizationPercentageCutoffLow(), delta); - Assert.assertEquals(67, dc.getDiskUtilizationSpaceCutoff()); + Assert.assertEquals(67, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(67, dc.getDiskUtilizationSpaceCutoffHigh()); dc = new DirectoryCollection(dirs, -57.5F, -57.5F, -67); Assert.assertEquals(0.0F, dc.getDiskUtilizationPercentageCutoffHigh(), delta); Assert.assertEquals(0.0F, dc.getDiskUtilizationPercentageCutoffLow(), delta); - Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoff()); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffHigh()); dc = new DirectoryCollection(dirs, 157.5F, 157.5F, -67); Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffHigh(), delta); Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffLow(), delta); - Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoff()); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(0, dc.getDiskUtilizationSpaceCutoffHigh()); + + dc = new DirectoryCollection(dirs, 157.5F, 157.5F, 5, 10); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffHigh(), + delta); + Assert.assertEquals(100.0F, dc.getDiskUtilizationPercentageCutoffLow(), + delta); + Assert.assertEquals(5, dc.getDiskUtilizationSpaceCutoffLow()); + Assert.assertEquals(10, dc.getDiskUtilizationSpaceCutoffHigh()); } @Test