From 2dc23bd9686bc59f2098e2195db0d105db414633 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Mon, 13 Jan 2020 10:38:25 -0800
Subject: [PATCH] Add protection in windows for slow file lock releasing
 (#50884)

This commit adds retries for windows cleanup after tests, which may fail
due to file locks not being immediately released after a windows process
exits.

closes #50825
---
 .../elasticsearch/packaging/util/Cleanup.java | 10 ++++----
 .../packaging/util/FileUtils.java             | 24 +++++++++++++++++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java
index f9b98d58cca..77384a38d27 100644
--- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java
+++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java
@@ -20,10 +20,12 @@
 package org.elasticsearch.packaging.util;
 
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.packaging.util.FileUtils.getTempDir;
 import static org.elasticsearch.packaging.util.FileUtils.lsGlob;
@@ -79,13 +81,13 @@ public class Cleanup {
 
         // delete files that may still exist
         lsGlob(getTempDir(), "elasticsearch*").forEach(FileUtils::rm);
-        final List<String> filesToDelete = Platforms.WINDOWS
-            ? ELASTICSEARCH_FILES_WINDOWS
-            : ELASTICSEARCH_FILES_LINUX;
+        final List<String> filesToDelete = Platforms.WINDOWS ? ELASTICSEARCH_FILES_WINDOWS : ELASTICSEARCH_FILES_LINUX;
+        // windows needs leniency due to asinine releasing of file locking async from a process exiting
+        Consumer<? super Path> rm = Platforms.WINDOWS ? FileUtils::rmWithRetries : FileUtils::rm;
         filesToDelete.stream()
             .map(Paths::get)
             .filter(Files::exists)
-            .forEach(FileUtils::rm);
+            .forEach(rm);
 
         // disable elasticsearch service
         // todo add this for windows when adding tests for service intallation
diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java
index efd2c85bf9e..2efb757037c 100644
--- a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java
+++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java
@@ -84,6 +84,30 @@ public class FileUtils {
         }
     }
 
+    public static void rmWithRetries(Path... paths) {
+        int tries = 10;
+        Exception exception = null;
+        while (tries-- > 0) {
+            try {
+                IOUtils.rm(paths);
+                return;
+            } catch (IOException e) {
+                if (exception == null) {
+                    exception = e;
+                } else {
+                    exception.addSuppressed(e);
+                }
+            }
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException interrupted) {
+                Thread.currentThread().interrupt();
+                return;
+            }
+        }
+        throw new RuntimeException(exception);
+    }
+
     public static Path mktempDir(Path path) {
         try {
             return Files.createTempDirectory(path,"tmp");