Discovery: don't wait joinThread when stopping

When a node stops, we cancel any ongoing join process. With #8327, we improved this logic and wait for it to complete before shutting down the node. However, the joining thread is part of a thread pool and will not stop until the thread pool is shutdown.

Another issue raised by the unneeded wait is that when we shutdown, we may ping ourselves - which results in an ugly warn level log. We now log all remote exception during pings at a debug level.

Closes #8359

src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java

@@ -253,6 +253,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
 
     @Override
     protected void doStop() throws ElasticsearchException {
+        joinThreadControl.stop();
         pingService.stop();
         masterFD.stop("zen disco stop");
         nodesFD.stop();
@@ -282,7 +283,6 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
                 }
             }
         }
-        joinThreadControl.stop();
     }
 
     @Override
@@ -1354,16 +1354,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
             running.set(false);
             Thread joinThread = currentJoinThread.getAndSet(null);
             if (joinThread != null) {
-                try {
+                joinThread.interrupt();
-                    joinThread.interrupt();
-                } catch (Exception e) {
-                    // ignore
-                }
-                try {
-                    joinThread.join(10000);
-                } catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
-                }
             }
         }
 

src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java

@@ -373,6 +373,9 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
                         } catch (ConnectTransportException e) {
                             // can't connect to the node - this is a more common path!
                             logger.trace("[{}] failed to connect to {}", e, sendPingsHandler.id(), finalNodeToSend);
+                        } catch (RemoteTransportException e) {
+                            // something went wrong on the other side
+                            logger.debug("[{}] received a remote error as a response to ping {}", e, sendPingsHandler.id(), finalNodeToSend);
                         } catch (Throwable e) {
                             logger.warn("[{}] failed send ping to {}", e, sendPingsHandler.id(), finalNodeToSend);
                         } finally {