Integration tests port fix, logs output, lib share changes (#5061)

* integration tests port fix, logs output, lib share changes

* Fix test compilation

* docker image fix

* Annotate DiscoverySelector.pick() as Nullable, use formatting instead of string concatenation for logging

* integration-tests logging check

* corrected integration-tests log location

* removed an experimental travis test exception

.travis.yml

@@ -42,3 +42,11 @@ matrix:
         - mvn install -q -ff -DskipTests -B
       script:
         - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
+      after_failure:
+        - for v in ~/shared/logs/*.log ; do
+          echo $v logtail ======================== ; tail -100 $v ;
+          done
+        - for v in broker middlemanager overlord router coordinator historical ; do
+          echo $v dmesg ======================== ;
+          docker exec -it druid-$v sh -c 'dmesg | tail -3' ;
+          done

integration-tests/docker/broker.conf

@@ -21,7 +21,7 @@ command=java
   -Ddruid.broker.cache.populateCache=true
   -Ddruid.cache.type=local
   -Ddruid.cache.sizeInBytes=40000000
-  -cp /usr/local/druid/lib/*
+  -cp /shared/docker/lib/*
   io.druid.cli.Main server broker
 redirect_stderr=true
 autorestart=false

integration-tests/docker/coordinator.conf

@@ -15,7 +15,7 @@ command=java
   -Ddruid.metadata.storage.connector.password=diurd
   -Ddruid.zk.service.host=druid-zookeeper-kafka
   -Ddruid.coordinator.startDelay=PT5S
-  -cp /usr/local/druid/lib/*
+  -cp /shared/docker/lib/*
   io.druid.cli.Main server coordinator
 redirect_stderr=true
 priority=100

integration-tests/docker/historical.conf

@@ -19,7 +19,7 @@ command=java
   -Ddruid.server.http.numThreads=100
   -Ddruid.segmentCache.locations="[{\"path\":\"/shared/druid/indexCache\",\"maxSize\":5000000000}]"
   -Ddruid.server.maxSize=5000000000
-  -cp /usr/local/druid/lib/*
+  -cp /shared/docker/lib/*
   io.druid.cli.Main server historical
 redirect_stderr=true
 priority=100

integration-tests/docker/middlemanager.conf

@@ -22,7 +22,7 @@ command=java
   -Ddruid.worker.ip=%(ENV_HOST_IP)s
   -Ddruid.selectors.indexing.serviceName=druid/overlord
   -Ddruid.indexer.task.chathandler.type=announce
-  -cp /usr/local/druid/lib/*
+  -cp /shared/docker/lib/*
   io.druid.cli.Main server middleManager
 redirect_stderr=true
 priority=100

integration-tests/docker/overlord.conf

@@ -17,7 +17,7 @@ command=java
   -Ddruid.indexer.storage.type=metadata
   -Ddruid.indexer.logs.directory=/shared/tasklogs
   -Ddruid.indexer.runner.type=remote
-  -cp /usr/local/druid/lib/*
+  -cp /shared/docker/lib/*
   io.druid.cli.Main server overlord
 redirect_stderr=true
 priority=100

integration-tests/docker/router.conf

@@ -10,7 +10,7 @@ command=java
   -Ddruid.host=%(ENV_HOST_IP)s
   -Ddruid.zk.service.host=druid-zookeeper-kafka
   -Ddruid.server.http.numThreads=100
-  -cp /usr/local/druid/lib/*
+  -cp /shared/docker/lib/*
   io.druid.cli.Main server router
 redirect_stderr=true
 priority=100

integration-tests/run_cluster.sh

@@ -28,25 +28,25 @@ mvn -B dependency:copy-dependencies -DoutputDirectory=$SHARED_DIR/docker/lib
 docker build -t druid/cluster $SHARED_DIR/docker
 
 # Start zookeeper and kafka
-docker run -d --name druid-zookeeper-kafka -p 2181:2181 -p 9092:9092 -v $SHARED_DIR:/shared -v $DOCKERDIR/zookeeper.conf:$SUPERVISORDIR/zookeeper.conf -v $DOCKERDIR/kafka.conf:$SUPERVISORDIR/kafka.conf druid/cluster
+docker run -d --privileged --name druid-zookeeper-kafka -p 2181:2181 -p 9092:9092 -v $SHARED_DIR:/shared -v $DOCKERDIR/zookeeper.conf:$SUPERVISORDIR/zookeeper.conf -v $DOCKERDIR/kafka.conf:$SUPERVISORDIR/kafka.conf druid/cluster
 
 # Start MYSQL 
-docker run -d --name druid-metadata-storage -v $SHARED_DIR:/shared -v $DOCKERDIR/metadata-storage.conf:$SUPERVISORDIR/metadata-storage.conf druid/cluster
+docker run -d --privileged --name druid-metadata-storage -v $SHARED_DIR:/shared -v $DOCKERDIR/metadata-storage.conf:$SUPERVISORDIR/metadata-storage.conf druid/cluster
 
 # Start Overlord
-docker run -d --name druid-overlord -p 8090:8090 -v $SHARED_DIR:/shared -v $DOCKERDIR/overlord.conf:$SUPERVISORDIR/overlord.conf --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
+docker run -d --privileged --name druid-overlord -p 8090:8090 -v $SHARED_DIR:/shared -v $DOCKERDIR/overlord.conf:$SUPERVISORDIR/overlord.conf --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
 
 # Start Coordinator
-docker run -d --name druid-coordinator -p 8081:8081 -v $SHARED_DIR:/shared -v $DOCKERDIR/coordinator.conf:$SUPERVISORDIR/coordinator.conf --link druid-overlord:druid-overlord --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
+docker run -d --privileged --name druid-coordinator -p 8081:8081 -v $SHARED_DIR:/shared -v $DOCKERDIR/coordinator.conf:$SUPERVISORDIR/coordinator.conf --link druid-overlord:druid-overlord --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
 
 # Start Historical 
-docker run -d --name druid-historical -v $SHARED_DIR:/shared -v $DOCKERDIR/historical.conf:$SUPERVISORDIR/historical.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
+docker run -d --privileged --name druid-historical -v $SHARED_DIR:/shared -v $DOCKERDIR/historical.conf:$SUPERVISORDIR/historical.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
 
 # Start Middlemanger
-docker run -d --name druid-middlemanager -p 8100:8100 -p 8101:8101 -p 8102:8102 -p 8103:8103 -p 8104:8104 -p 8105:8105 -v $RESOURCEDIR:/resources -v $SHARED_DIR:/shared -v $DOCKERDIR/middlemanager.conf:$SUPERVISORDIR/middlemanager.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-overlord:druid-overlord druid/cluster
+docker run -d --privileged --name druid-middlemanager -p 8091:8091 -p 8100:8100 -p 8101:8101 -p 8102:8102 -p 8103:8103 -p 8104:8104 -p 8105:8105 -v $RESOURCEDIR:/resources -v $SHARED_DIR:/shared -v $DOCKERDIR/middlemanager.conf:$SUPERVISORDIR/middlemanager.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-overlord:druid-overlord druid/cluster
 
 # Start Broker 
-docker run -d --name druid-broker -p 8082:8082 -v $SHARED_DIR:/shared -v $DOCKERDIR/broker.conf:$SUPERVISORDIR/broker.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-middlemanager:druid-middlemanager --link druid-historical:druid-historical druid/cluster
+docker run -d --privileged --name druid-broker -p 8082:8082 -v $SHARED_DIR:/shared -v $DOCKERDIR/broker.conf:$SUPERVISORDIR/broker.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-middlemanager:druid-middlemanager --link druid-historical:druid-historical druid/cluster
 
 # Start Router 
-docker run -d --name druid-router -p 8888:8888 -v $SHARED_DIR:/shared -v $DOCKERDIR/router.conf:$SUPERVISORDIR/router.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster
+docker run -d --privileged --name druid-router -p 8888:8888 -v $SHARED_DIR:/shared -v $DOCKERDIR/router.conf:$SUPERVISORDIR/router.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster

server/src/main/java/io/druid/client/selector/AbstractTierSelectorStrategy.java

@@ -23,6 +23,7 @@ import com.google.common.collect.Iterables;
 import io.druid.timeline.DataSegment;
 import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap;
 
+import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -38,6 +39,7 @@ public abstract class AbstractTierSelectorStrategy implements TierSelectorStrate
     this.serverSelectorStrategy = serverSelectorStrategy;
   }
 
+  @Nullable
   @Override
   public QueryableDruidServer pick(
       Int2ObjectRBTreeMap<Set<QueryableDruidServer>> prioritizedServers,

server/src/main/java/io/druid/client/selector/DiscoverySelector.java

@@ -19,9 +19,12 @@
 
 package io.druid.client.selector;
 
+import javax.annotation.Nullable;
+
 /**
  */
 public interface DiscoverySelector<T>
 {
+  @Nullable
   T pick();
 }

server/src/main/java/io/druid/client/selector/ServerSelector.java

@@ -23,6 +23,7 @@ import io.druid.server.coordination.DruidServerMetadata;
 import io.druid.timeline.DataSegment;
 import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap;
 
+import javax.annotation.Nullable;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
@@ -110,6 +111,7 @@ public class ServerSelector implements DiscoverySelector<QueryableDruidServer>
     }
   }
 
+  @Nullable
   @Override
   public QueryableDruidServer pick()
   {

server/src/main/java/io/druid/client/selector/TierSelectorStrategy.java

@@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import io.druid.timeline.DataSegment;
 import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap;
 
+import javax.annotation.Nullable;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
@@ -40,6 +41,7 @@ public interface TierSelectorStrategy
 {
   Comparator<Integer> getComparator();
 
+  @Nullable
   QueryableDruidServer pick(Int2ObjectRBTreeMap<Set<QueryableDruidServer>> prioritizedServers, DataSegment segment);
 
   List<QueryableDruidServer> pick(

server/src/main/java/io/druid/curator/discovery/ServerDiscoveryFactory.java

@@ -46,14 +46,14 @@ public class ServerDiscoveryFactory
   public ServerDiscoverySelector createSelector(String serviceName)
   {
     if (serviceName == null) {
-      return new ServerDiscoverySelector(new NoopServiceProvider());
+      return new ServerDiscoverySelector(new NoopServiceProvider(), serviceName);
     }
 
     final ServiceProvider serviceProvider = serviceDiscovery
         .serviceProviderBuilder()
         .serviceName(CuratorServiceUtils.makeCanonicalServiceName(serviceName))
         .build();
-    return new ServerDiscoverySelector(serviceProvider);
+    return new ServerDiscoverySelector(serviceProvider, serviceName);
   }
 
   private static class NoopServiceProvider<T> implements ServiceProvider<T>

server/src/main/java/io/druid/curator/discovery/ServerDiscoverySelector.java

@@ -31,6 +31,7 @@ import io.druid.java.util.common.logger.Logger;
 import org.apache.curator.x.discovery.ServiceInstance;
 import org.apache.curator.x.discovery.ServiceProvider;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
@@ -44,10 +45,12 @@ public class ServerDiscoverySelector implements DiscoverySelector<Server>
   private static final Logger log = new Logger(ServerDiscoverySelector.class);
 
   private final ServiceProvider serviceProvider;
+  private final String name;
 
-  public ServerDiscoverySelector(ServiceProvider serviceProvider)
+  public ServerDiscoverySelector(ServiceProvider serviceProvider, String name)
   {
     this.serviceProvider = serviceProvider;
+    this.name = name;
   }
 
   private static final Function<ServiceInstance, Server> TO_SERVER = new Function<ServiceInstance, Server>()
@@ -97,6 +100,7 @@ public class ServerDiscoverySelector implements DiscoverySelector<Server>
     }
   };
 
+  @Nullable
   @Override
   public Server pick()
   {
@@ -105,12 +109,12 @@ public class ServerDiscoverySelector implements DiscoverySelector<Server>
       instance = serviceProvider.getInstance();
     }
     catch (Exception e) {
-      log.info(e, "Exception getting instance");
+      log.info(e, "Exception getting instance for [%s]", name);
       return null;
     }
 
     if (instance == null) {
-      log.error("No server instance found");
+      log.error("No server instance found for [%s]", name);
       return null;
     }
 

server/src/test/java/io/druid/curator/discovery/ServerDiscoverySelectorTest.java

@@ -46,7 +46,7 @@ public class ServerDiscoverySelectorTest
   {
     serviceProvider = EasyMock.createMock(ServiceProvider.class);
     instance = EasyMock.createMock(ServiceInstance.class);
-    serverDiscoverySelector = new ServerDiscoverySelector(serviceProvider);
+    serverDiscoverySelector = new ServerDiscoverySelector(serviceProvider, "test");
   }
 
   @Test