From 3ba0fd1e50192336d6a0125768739973972917cd Mon Sep 17 00:00:00 2001
From: Eric E Payne <ericp@verizonmedia.com>
Date: Thu, 9 Jan 2020 17:18:44 +0000
Subject: [PATCH] YARN-9018. Add functionality to AuxiliaryLocalPathHandler to
 return all locations to read for a given path. Contributed by Kuhu Shukla
 (kshukla)

(cherry picked from commit 93233a7d6e4d6b8098622a1aa830355cc18d9589)
---
 .../java/org/apache/hadoop/mapred/TestShuffleHandler.java | 8 ++++++++
 .../hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java | 8 ++++++++
 .../yarn/server/nodemanager/LocalDirsHandlerService.java  | 4 ++++
 .../containermanager/ContainerManagerImpl.java            | 5 +++++
 4 files changed, 25 insertions(+)

diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 801724851d3..698821908e9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -173,6 +173,14 @@ public class TestShuffleHandler {
         throws IOException {
       return new Path(ABS_LOG_DIR.getAbsolutePath());
     }
+
+    @Override
+    public Iterable<Path> getAllLocalPathsForRead(String path)
+        throws IOException {
+      ArrayList<Path> paths = new ArrayList<>();
+      paths.add(new Path(ABS_LOG_DIR.getAbsolutePath()));
+      return paths;
+    }
   }
 
   private static class MockShuffleHandler2 extends
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
index 50feecf4535..452edce0f0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
@@ -55,4 +55,12 @@ public interface AuxiliaryLocalPathHandler {
    * @throws IOException if the path creations fails
    */
   Path getLocalPathForWrite(String path, long size) throws IOException;
+
+  /**
+   * Get all paths from the local FS for reading for a given Auxiliary Service.
+   * @param path the requested path
+   * @return the complete path list to the file on a local disk as an Iterable
+   * @throws IOException if the file read encounters a problem
+   */
+  Iterable<Path> getAllLocalPathsForRead(String path) throws IOException;
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index ac0b84d1b95..9af16455b60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -624,6 +624,10 @@ public class LocalDirsHandlerService extends AbstractService {
     return getPathToRead(pathStr, getLocalDirsForRead());
   }
 
+  public Iterable<Path> getAllLocalPathsForRead(String pathStr) throws IOException {
+    return localDirsAllocator.getAllLocalPathsToRead(pathStr, getConfig());
+  }
+
   public Path getLogPathForWrite(String pathStr, boolean checkWrite)
       throws IOException {
     return logDirsAllocator.getLocalPathForWrite(pathStr,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 8afc52ab7ba..0feb3c8e621 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -1644,6 +1644,11 @@ public class ContainerManagerImpl extends CompositeService implements
         throws IOException {
       return dirhandlerService.getLocalPathForWrite(path, size, false);
     }
+
+    @Override
+    public Iterable<Path> getAllLocalPathsForRead(String path) throws IOException {
+      return dirhandlerService.getAllLocalPathsForRead(path);
+    }
   }
 
   @SuppressWarnings("unchecked")