More Verbose Assertion in testSnapshotWithStuckNode (#39893) (#39928)

* The test failure in #39852 is caused by a file in the initial repository when there should not be any
  * It seems that on a normal consistent file system no left-over file should exist ever here after the validation finishes and I can't reproduce or see any other path to a dangling file in the fresh respository
=> added a more verbose and strict assertion that will log what file is left over next time
* Relates #39852

server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java

@@ -35,6 +35,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
@@ -74,6 +75,18 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase {
         return failureCount;
     }
 
+    public static void assertFileCount(Path dir, int expectedCount) throws IOException {
+        final List<Path> found = new ArrayList<>();
+        Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+                found.add(file);
+                return FileVisitResult.CONTINUE;
+            }
+        });
+        assertEquals("Unexpected file count, found: [" + found + "].", expectedCount, found.size());
+    }
+
     public static int numberOfFiles(Path dir) throws IOException {
         final AtomicInteger count = new AtomicInteger();
         Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {

server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java

@@ -454,7 +454,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
         // Remove it from the list of available nodes
         nodes.remove(blockedNode);
 
-        int numberOfFilesBeforeSnapshot = numberOfFiles(repo);
+        assertFileCount(repo, 0);
         logger.info("--> snapshot");
         client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
                                 .setWaitForCompletion(false)
@@ -490,8 +490,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
         //   (2) index-0 (because we keep the previous version) and
         //   (3) index-latest
         //   (4) incompatible-snapshots
-        assertThat("not all files were deleted during snapshot cancellation",
+        assertFileCount(repo, 4);
-            numberOfFilesBeforeSnapshot, equalTo(numberOfFiles(repo) - 4));
         logger.info("--> done");
     }