Handle long paths on Windows for standalone tests

In some cases our Windows builds fail due to long path names that arise
from a combination of long build job names plus long sub-project
names. While newer versions of Windows can handle long paths, invoking
batch scripts longer than 260 characters via cmd.exe is still
problematic. This leads to failing integration tests because we can not
run the commands to install plugins, create the keystore, and start the
node. This commit handles this by converting all paths on Windows used
to start an Elasticsearch node to short path names.

Relates #26365

buildSrc/build.gradle

@@ -94,6 +94,7 @@ dependencies {
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
   compile 'de.thetaphi:forbiddenapis:2.3'
   compile 'org.apache.rat:apache-rat:0.11'
+  compile "org.elasticsearch:jna:4.4.0-1"
 }
 
 // Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -206,7 +206,19 @@ class ClusterFormationTasks {
         for (Map.Entry<String, Object[]> command : node.config.setupCommands.entrySet()) {
             // the first argument is the actual script name, relative to home
             Object[] args = command.getValue().clone()
-            args[0] = new File(node.homeDir, args[0].toString())
+            final Object commandPath
+            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+                /*
+                 * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
+                 * getting the short name requiring the path to already exist. Note that we have to capture the value of arg[0] now
+                 * otherwise we would stack overflow later since arg[0] is replaced below.
+                 */
+                String argsZero = args[0]
+                commandPath = "${-> Paths.get(NodeInfo.getShortPathName(node.homeDir.toString())).resolve(argsZero.toString()).toString()}"
+            } else {
+                commandPath = node.homeDir.toPath().resolve(args[0].toString()).toString()
+            }
+            args[0] = commandPath
             setup = configureExecTask(taskName(prefix, node, command.getKey()), project, setup, node, args)
         }
 
@@ -337,7 +349,11 @@ class ClusterFormationTasks {
         if (node.config.keystoreSettings.isEmpty()) {
             return setup
         } else {
-            File esKeystoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile()
+            /*
+             * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
+             * getting the short name requiring the path to already exist.
+             */
+            final Object esKeystoreUtil = "${-> node.binPath().resolve('elasticsearch-keystore').toString()}"
             return configureExecTask(name, project, setup, node, esKeystoreUtil, 'create')
         }
     }
@@ -345,8 +361,12 @@ class ClusterFormationTasks {
     /** Adds tasks to add settings to the keystore */
     static Task configureAddKeystoreSettingTasks(String parent, Project project, Task setup, NodeInfo node) {
         Map kvs = node.config.keystoreSettings
-        File esKeystoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile()
         Task parentTask = setup
+        /*
+         * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to getting
+         * the short name requiring the path to already exist.
+         */
+        final Object esKeystoreUtil = "${-> node.binPath().resolve('elasticsearch-keystore').toString()}"
         for (Map.Entry<String, String> entry in kvs) {
             String key = entry.getKey()
             String name = taskName(parent, node, 'addToKeystore#' + key)
@@ -482,8 +502,13 @@ class ClusterFormationTasks {
             pluginZip = project.configurations.getByName("_plugin_${prefix}_${plugin.path}")
         }
         // delay reading the file location until execution time by wrapping in a closure within a GString
-        Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
+        final Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
-        Object[] args = [new File(node.homeDir, 'bin/elasticsearch-plugin'), 'install', file]
+        /*
+         * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to getting
+         * the short name requiring the path to already exist.
+         */
+        final Object esPluginUtil = "${-> node.binPath().resolve('elasticsearch-plugin').toString()}"
+        final Object[] args = [esPluginUtil, 'install', file]
         return configureExecTask(name, project, setup, node, args)
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/JNAKernel32Library.java

@@ -0,0 +1,25 @@
+package org.elasticsearch.gradle.test;
+
+import com.sun.jna.Native;
+import com.sun.jna.WString;
+import org.apache.tools.ant.taskdefs.condition.Os;
+
+public class JNAKernel32Library {
+
+    private static final class Holder {
+        private static final JNAKernel32Library instance = new JNAKernel32Library();
+    }
+
+    static JNAKernel32Library getInstance() {
+        return Holder.instance;
+    }
+
+    private JNAKernel32Library() {
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            Native.register("kernel32");
+        }
+    }
+
+    native int GetShortPathNameW(WString lpszLongPath, char[] lpszShortPath, int cchBuffer);
+
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -18,11 +18,16 @@
  */
 package org.elasticsearch.gradle.test
 
+import com.sun.jna.Native
+import com.sun.jna.WString
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.Version
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 
+import java.nio.file.Path
+import java.nio.file.Paths
+
 /**
  * A container for the files and configuration associated with a single node in a test cluster.
  */
@@ -85,10 +90,10 @@ class NodeInfo {
     String executable
 
     /** Path to the elasticsearch start script */
-    File esScript
+    private Object esScript
 
     /** script to run when running in the background */
-    File wrapperScript
+    private File wrapperScript
 
     /** buffer for ant output when starting this node */
     ByteArrayOutputStream buffer = new ByteArrayOutputStream()
@@ -132,11 +137,15 @@ class NodeInfo {
             args.add('/C')
             args.add('"') // quote the entire command
             wrapperScript = new File(cwd, "run.bat")
-            esScript = new File(homeDir, 'bin/elasticsearch.bat')
+            /*
+             * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
+             * getting the short name requiring the path to already exist.
+             */
+            esScript = "${-> binPath().resolve('elasticsearch.bat').toString()}"
         } else {
             executable = 'bash'
             wrapperScript = new File(cwd, "run")
-            esScript = new File(homeDir, 'bin/elasticsearch')
+            esScript = binPath().resolve('elasticsearch')
         }
         if (config.daemonize) {
             args.add("${wrapperScript}")
@@ -170,6 +179,31 @@ class NodeInfo {
         }
     }
 
+    Path binPath() {
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            return Paths.get(getShortPathName(new File(homeDir, 'bin').toString()))
+        } else {
+            return Paths.get(new File(homeDir, 'bin').toURI())
+        }
+    }
+
+    static String getShortPathName(String path) {
+        assert Os.isFamily(Os.FAMILY_WINDOWS)
+        final WString longPath = new WString("\\\\?\\" + path)
+        // first we get the length of the buffer needed
+        final int length = JNAKernel32Library.getInstance().GetShortPathNameW(longPath, null, 0)
+        if (length == 0) {
+            throw new IllegalStateException("path [" + path + "] encountered error [" + Native.getLastError() + "]")
+        }
+        final char[] shortPath = new char[length]
+        // knowing the length of the buffer, now we get the short name
+        if (JNAKernel32Library.getInstance().GetShortPathNameW(longPath, shortPath, length) == 0) {
+            throw new IllegalStateException("path [" + path + "] encountered error [" + Native.getLastError() + "]")
+        }
+        // we have to strip the \\?\ away from the path for cmd.exe
+        return Native.toString(shortPath).substring(4)
+    }
+
     /** Returns debug string for the command that started this node. */
     String getCommandString() {
         String esCommandString = "\nNode ${nodeNum} configuration:\n"

buildSrc/version.properties

@@ -10,6 +10,8 @@ snakeyaml         = 1.15
 # When updating log4j, please update also docs/java-api/index.asciidoc
 log4j             = 2.8.2
 slf4j             = 1.6.2
+
+# when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 4.4.0-1
 
 # test dependencies