From 97866eb0e874590d08e781f49f643024928668a1 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 6 Apr 2017 14:42:27 +0100 Subject: [PATCH] [ML] Increase timeouts for named pipe connections to native processes (elastic/x-pack-elasticsearch#987) It has been observed that Amazon EBS volumes created from snapshots can have very high latency the first time a given block is accessed. This can lead to named pipes taking longer than 2 seconds to create. Since the native processes create their named pipes immediately after startup, and this only takes a fraction of a second on a local disk, 2 seconds was considered a generous timeout, but it seems that in the case of a remote NAS with lazy provisioning it's not long enough. During debugging a latency of just over 3 seconds was observed. The timeouts have been increased to 10 seconds. relates elastic/x-pack-elasticsearch#922 Original commit: elastic/x-pack-elasticsearch@c90434c948e82212270ea6f9565c5b9d98d4b041 --- .../elasticsearch/xpack/ml/job/process/NativeController.java | 5 +++-- .../process/autodetect/NativeAutodetectProcessFactory.java | 2 +- .../process/normalizer/NativeNormalizerProcessFactory.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java index 20eae2ebf32..da44cb65196 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java @@ -31,8 +31,9 @@ import java.util.regex.Pattern; public class NativeController { private static final Logger LOGGER = Loggers.getLogger(NativeController.class); - // The controller process should already be running by the time this class tries to connect to it, so the timeout can be short - private static final Duration CONTROLLER_CONNECT_TIMEOUT = Duration.ofSeconds(2); + // The controller process should already be running by the time this class tries to connect to it, so the timeout + // can be short (although there's a gotcha with EBS volumes restored from snapshot, so not TOO short) + private static final Duration CONTROLLER_CONNECT_TIMEOUT = Duration.ofSeconds(10); private static final String START_COMMAND = "start"; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java index c9d8a0c8b15..8ff9808a823 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java @@ -40,7 +40,7 @@ public class NativeAutodetectProcessFactory implements AutodetectProcessFactory private static final Logger LOGGER = Loggers.getLogger(NativeAutodetectProcessFactory.class); private static final NamedPipeHelper NAMED_PIPE_HELPER = new NamedPipeHelper(); - private static final Duration PROCESS_STARTUP_TIMEOUT = Duration.ofSeconds(2); + private static final Duration PROCESS_STARTUP_TIMEOUT = Duration.ofSeconds(10); private final Client client; private final Environment env; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java index d0974cb377c..8d68b305f02 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java @@ -26,7 +26,7 @@ public class NativeNormalizerProcessFactory implements NormalizerProcessFactory private static final Logger LOGGER = Loggers.getLogger(NativeNormalizerProcessFactory.class); private static final NamedPipeHelper NAMED_PIPE_HELPER = new NamedPipeHelper(); - private static final Duration PROCESS_STARTUP_TIMEOUT = Duration.ofSeconds(2); + private static final Duration PROCESS_STARTUP_TIMEOUT = Duration.ofSeconds(10); private final Environment env; private final Settings settings;