honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Handle long paths on Windows for standalone tests 

In some cases our Windows builds fail due to long path names that arise
from a combination of long build job names plus long sub-project
names. While newer versions of Windows can handle long paths, invoking
batch scripts longer than 260 characters via cmd.exe is still
problematic. This leads to failing integration tests because we can not
run the commands to install plugins, create the keystore, and start the
node. This commit handles this by converting all paths on Windows used
to start an Elasticsearch node to short path names.

Relates #26365
This commit is contained in:
Jason Tedor 2017-08-24 18:46:49 -04:00 committed by GitHub
parent 5202e7e93b
commit 911e1f6203
5 changed files with 96 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
buildSrc/build.gradle
View File

@ -94,6 +94,7 @@ dependencies {
compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
compile 'de.thetaphi:forbiddenapis:2.3'
compile 'org.apache.rat:apache-rat:0.11'
compile "org.elasticsearch:jna:4.4.0-1"
}
// Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs

35
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View File

@ -206,7 +206,19 @@ class ClusterFormationTasks {
for (Map.Entry<String, Object[]> command : node.config.setupCommands.entrySet()) {
// the first argument is the actual script name, relative to home
Object[] args = command.getValue().clone()
args[0] = new File(node.homeDir, args[0].toString())
final Object commandPath
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
/*
* We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
* getting the short name requiring the path to already exist. Note that we have to capture the value of arg[0] now
* otherwise we would stack overflow later since arg[0] is replaced below.
*/
String argsZero = args[0]
commandPath = "${-> Paths.get(NodeInfo.getShortPathName(node.homeDir.toString())).resolve(argsZero.toString()).toString()}"
} else {
commandPath = node.homeDir.toPath().resolve(args[0].toString()).toString()
}
args[0] = commandPath
setup = configureExecTask(taskName(prefix, node, command.getKey()), project, setup, node, args)
}
@ -337,7 +349,11 @@ class ClusterFormationTasks {
if (node.config.keystoreSettings.isEmpty()) {
return setup
} else {
File esKeystoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile()
/*
* We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
* getting the short name requiring the path to already exist.
*/
final Object esKeystoreUtil = "${-> node.binPath().resolve('elasticsearch-keystore').toString()}"
return configureExecTask(name, project, setup, node, esKeystoreUtil, 'create')
}
}
@ -345,8 +361,12 @@ class ClusterFormationTasks {
/** Adds tasks to add settings to the keystore */
static Task configureAddKeystoreSettingTasks(String parent, Project project, Task setup, NodeInfo node) {
Map kvs = node.config.keystoreSettings
File esKeystoreUtil = Paths.get(node.homeDir.toString(), "bin/" + "elasticsearch-keystore").toFile()
Task parentTask = setup
/*
* We have to delay building the string as the path will not exist during configuration which will fail on Windows due to getting
* the short name requiring the path to already exist.
*/
final Object esKeystoreUtil = "${-> node.binPath().resolve('elasticsearch-keystore').toString()}"
for (Map.Entry<String, String> entry in kvs) {
String key = entry.getKey()
String name = taskName(parent, node, 'addToKeystore#' + key)
@ -482,8 +502,13 @@ class ClusterFormationTasks {
pluginZip = project.configurations.getByName("_plugin_${prefix}_${plugin.path}")
}
// delay reading the file location until execution time by wrapping in a closure within a GString
Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
Object[] args = [new File(node.homeDir, 'bin/elasticsearch-plugin'), 'install', file]
final Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
/*
* We have to delay building the string as the path will not exist during configuration which will fail on Windows due to getting
* the short name requiring the path to already exist.
*/
final Object esPluginUtil = "${-> node.binPath().resolve('elasticsearch-plugin').toString()}"
final Object[] args = [esPluginUtil, 'install', file]
return configureExecTask(name, project, setup, node, args)
}

25
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/JNAKernel32Library.java Normal file
View File

@ -0,0 +1,25 @@
package org.elasticsearch.gradle.test;
import com.sun.jna.Native;
import com.sun.jna.WString;
import org.apache.tools.ant.taskdefs.condition.Os;
public class JNAKernel32Library {
private static final class Holder {
private static final JNAKernel32Library instance = new JNAKernel32Library();
}
static JNAKernel32Library getInstance() {
return Holder.instance;
}
private JNAKernel32Library() {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
Native.register("kernel32");
}
}
native int GetShortPathNameW(WString lpszLongPath, char[] lpszShortPath, int cchBuffer);
}

42
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
View File

@ -18,11 +18,16 @@
*/
package org.elasticsearch.gradle.test
import com.sun.jna.Native
import com.sun.jna.WString
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.Version
import org.gradle.api.InvalidUserDataException
import org.gradle.api.Project
import java.nio.file.Path
import java.nio.file.Paths
/**
* A container for the files and configuration associated with a single node in a test cluster.
*/
@ -85,10 +90,10 @@ class NodeInfo {
String executable
/** Path to the elasticsearch start script */
File esScript
private Object esScript
/** script to run when running in the background */
File wrapperScript
private File wrapperScript
/** buffer for ant output when starting this node */
ByteArrayOutputStream buffer = new ByteArrayOutputStream()
@ -132,11 +137,15 @@ class NodeInfo {
args.add('/C')
args.add('"') // quote the entire command
wrapperScript = new File(cwd, "run.bat")
esScript = new File(homeDir, 'bin/elasticsearch.bat')
/*
* We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
* getting the short name requiring the path to already exist.
*/
esScript = "${-> binPath().resolve('elasticsearch.bat').toString()}"
} else {
executable = 'bash'
wrapperScript = new File(cwd, "run")
esScript = new File(homeDir, 'bin/elasticsearch')
esScript = binPath().resolve('elasticsearch')
}
if (config.daemonize) {
args.add("${wrapperScript}")
@ -170,6 +179,31 @@ class NodeInfo {
}
}
Path binPath() {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
return Paths.get(getShortPathName(new File(homeDir, 'bin').toString()))
} else {
return Paths.get(new File(homeDir, 'bin').toURI())
}
}
static String getShortPathName(String path) {
assert Os.isFamily(Os.FAMILY_WINDOWS)
final WString longPath = new WString("\\\\?\\" + path)
// first we get the length of the buffer needed
final int length = JNAKernel32Library.getInstance().GetShortPathNameW(longPath, null, 0)
if (length == 0) {
throw new IllegalStateException("path [" + path + "] encountered error [" + Native.getLastError() + "]")
}
final char[] shortPath = new char[length]
// knowing the length of the buffer, now we get the short name
if (JNAKernel32Library.getInstance().GetShortPathNameW(longPath, shortPath, length) == 0) {
throw new IllegalStateException("path [" + path + "] encountered error [" + Native.getLastError() + "]")
}
// we have to strip the \\?\ away from the path for cmd.exe
return Native.toString(shortPath).substring(4)
}
/** Returns debug string for the command that started this node. */
String getCommandString() {
String esCommandString = "\nNode ${nodeNum} configuration:\n"

2
buildSrc/version.properties
View File

@ -10,6 +10,8 @@ snakeyaml = 1.15
# When updating log4j, please update also docs/java-api/index.asciidoc
log4j = 2.8.2
slf4j = 1.6.2
# when updating the JNA version, also update the version in buildSrc/build.gradle
jna = 4.4.0-1
# test dependencies