From f1e180902956fd547ad6a7b440588fdf583870c2 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Sat, 1 May 2021 19:10:32 -0700 Subject: [PATCH] HADOOP-17653. Do not use guava's Files.createTempDir(). (#2945) Reviewed-by: Steve Loughran Signed-off-by: Akira Ajisaka --- .../store/driver/impl/StateStoreFileImpl.java | 13 ++++++++++++- .../impl/TestInMemoryLevelDBAliasMapClient.java | 9 ++++++--- .../impl/TestLevelDbMockAliasMapClient.java | 7 +++++-- .../hadoop/yarn/service/client/ServiceClient.java | 7 +++++-- .../hadoop/yarn/csi/client/TestCsiClient.java | 7 +++++-- .../resourceplugin/com/nec/TestNECVEPlugin.java | 2 +- .../fpga/TestIntelFpgaOpenclPlugin.java | 2 +- .../TestDominantResourceFairnessPolicy.java | 2 +- 8 files changed, 36 insertions(+), 13 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java index cedc784e39f..9d2b1ab2fb7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java @@ -88,7 +88,18 @@ public class StateStoreFileImpl extends StateStoreFileBaseImpl { if (this.rootDirectory == null) { String dir = getConf().get(FEDERATION_STORE_FILE_DIRECTORY); if (dir == null) { - File tempDir = Files.createTempDir(); + File tempDirBase = + new File(System.getProperty("java.io.tmpdir")); + File tempDir = null; + try { + tempDir = java.nio.file.Files.createTempDirectory( + tempDirBase.toPath(), System.currentTimeMillis() + "-").toFile(); + } catch (IOException e) { + // fallback to the base upon exception. + LOG.debug("Unable to create a temporary directory. Fall back to " + + " the default system temp directory {}", tempDirBase, e); + tempDir = tempDirBase; + } dir = tempDir.getAbsolutePath(); LOG.warn("The root directory is not available, using {}", dir); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java index 8971730707b..1996f4455af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java @@ -16,8 +16,6 @@ */ package org.apache.hadoop.hdfs.server.common.blockaliasmap.impl; -import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; -import org.apache.hadoop.thirdparty.com.google.common.io.Files; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -28,7 +26,9 @@ import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap; import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer; import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap; import org.apache.hadoop.hdfs.server.common.FileRegion; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -44,6 +44,7 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.List; import java.util.Optional; import java.util.Random; @@ -74,7 +75,9 @@ public class TestInMemoryLevelDBAliasMapClient { conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS, "localhost:" + port); - tempDir = Files.createTempDir(); + File testDir = GenericTestUtils.getTestDir(); + tempDir = Files + .createTempDirectory(testDir.toPath(), "test").toFile(); File levelDBDir = new File(tempDir, BPID); levelDBDir.mkdirs(); conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java index 91e07cfa4c8..84b0d23fda5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java @@ -16,7 +16,6 @@ */ package org.apache.hadoop.hdfs.server.common.blockaliasmap.impl; -import org.apache.hadoop.thirdparty.com.google.common.io.Files; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -26,12 +25,14 @@ import org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation; import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap; import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer; import org.apache.hadoop.hdfs.server.common.FileRegion; +import org.apache.hadoop.test.GenericTestUtils; import org.iq80.leveldb.DBException; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; import static org.mockito.Mockito.doThrow; @@ -60,7 +61,9 @@ public class TestLevelDbMockAliasMapClient { conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS, "localhost:" + port); - tempDir = Files.createTempDir(); + File testDir = GenericTestUtils.getTestDir(); + tempDir = Files + .createTempDirectory(testDir.toPath(), "test").toFile(); conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, tempDir.getAbsolutePath()); levelDBAliasMapServer.setConf(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 901b81fe149..8d350158fff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -27,7 +27,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.RetryNTimes; -import org.apache.curator.shaded.com.google.common.io.Files; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -117,6 +116,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.text.MessageFormat; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -1140,7 +1140,10 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, return; } String buffer = ServiceApiUtil.jsonSerDeser.toJson(app); - File tmpDir = Files.createTempDir(); + File testDir = + new File(System.getProperty("java.io.tmpdir")); + File tmpDir = Files.createTempDirectory( + testDir.toPath(), System.currentTimeMillis() + "-").toFile(); if (tmpDir.exists()) { String serviceJsonPath = tmpDir.getAbsolutePath() + "/app.json"; File localFile = new File(serviceJsonPath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java index f025bc6d72f..b60d5cdb860 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java @@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.csi.client; import csi.v0.Csi; import org.apache.commons.io.FileUtils; -import org.apache.hadoop.thirdparty.com.google.common.io.Files; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Assume; @@ -30,6 +30,7 @@ import org.junit.Test; import java.io.File; import java.io.IOException; +import java.nio.file.Files; /** * Test class for CSI client. @@ -42,7 +43,9 @@ public class TestCsiClient { @BeforeClass public static void setUp() throws IOException { - testRoot = Files.createTempDir(); + File testDir = GenericTestUtils.getTestDir(); + testRoot = Files + .createTempDirectory(testDir.toPath(), "test").toFile(); File socketPath = new File(testRoot, "csi.sock"); FileUtils.forceMkdirParent(socketPath); domainSocket = "unix://" + socketPath.getAbsolutePath(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java index 06d1d2de6f9..58fc4c7dc1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java @@ -46,7 +46,7 @@ import java.util.function.Function; import org.apache.commons.compress.utils.Sets; import org.apache.commons.io.FileUtils; -import org.apache.curator.shaded.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.util.Shell.CommandExecutor; import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestIntelFpgaOpenclPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestIntelFpgaOpenclPlugin.java index 663df5e26cb..0b093953eef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestIntelFpgaOpenclPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestIntelFpgaOpenclPlugin.java @@ -22,7 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.curator.shaded.com.google.common.collect.Lists; +import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.fs.Path; import org.junit.Before; import org.junit.Test; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java index c963e0dab63..197651a64c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.Map; import java.util.TreeSet; -import org.apache.curator.shaded.com.google.common.base.Joiner; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation;