Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Integration tests port fix, logs output, lib share changes (#5061) 

* integration tests port fix, logs output, lib share changes

* Fix test compilation

* docker image fix

* Annotate DiscoverySelector.pick() as Nullable, use formatting instead of string concatenation for logging

* integration-tests logging check

* corrected integration-tests log location

* removed an experimental travis test exception
This commit is contained in:
Egor Riashin 2017-11-11 01:07:37 +03:00 committed by Slim
parent 9444da5038
commit 3ad5f51f7e
15 changed files with 41 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.travis.yml
View File

@ -42,3 +42,11 @@ matrix:
- mvn install -q -ff -DskipTests -B
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
after_failure:
- for v in ~/shared/logs/*.log ; do
echo $v logtail ======================== ; tail -100 $v ;
done
- for v in broker middlemanager overlord router coordinator historical ; do
echo $v dmesg ======================== ;
docker exec -it druid-$v sh -c 'dmesg | tail -3' ;
done

2
integration-tests/docker/broker.conf
View File

@ -21,7 +21,7 @@ command=java
-Ddruid.broker.cache.populateCache=true
-Ddruid.cache.type=local
-Ddruid.cache.sizeInBytes=40000000
-cp /usr/local/druid/lib/*
-cp /shared/docker/lib/*
io.druid.cli.Main server broker
redirect_stderr=true
autorestart=false

2
integration-tests/docker/coordinator.conf
View File

@ -15,7 +15,7 @@ command=java
-Ddruid.metadata.storage.connector.password=diurd
-Ddruid.zk.service.host=druid-zookeeper-kafka
-Ddruid.coordinator.startDelay=PT5S
-cp /usr/local/druid/lib/*
-cp /shared/docker/lib/*
io.druid.cli.Main server coordinator
redirect_stderr=true
priority=100

2
integration-tests/docker/historical.conf
View File

@ -19,7 +19,7 @@ command=java
-Ddruid.server.http.numThreads=100
-Ddruid.segmentCache.locations="[{\"path\":\"/shared/druid/indexCache\",\"maxSize\":5000000000}]"
-Ddruid.server.maxSize=5000000000
-cp /usr/local/druid/lib/*
-cp /shared/docker/lib/*
io.druid.cli.Main server historical
redirect_stderr=true
priority=100

2
integration-tests/docker/middlemanager.conf
View File

@ -22,7 +22,7 @@ command=java
-Ddruid.worker.ip=%(ENV_HOST_IP)s
-Ddruid.selectors.indexing.serviceName=druid/overlord
-Ddruid.indexer.task.chathandler.type=announce
-cp /usr/local/druid/lib/*
-cp /shared/docker/lib/*
io.druid.cli.Main server middleManager
redirect_stderr=true
priority=100

2
integration-tests/docker/overlord.conf
View File

@ -17,7 +17,7 @@ command=java
-Ddruid.indexer.storage.type=metadata
-Ddruid.indexer.logs.directory=/shared/tasklogs
-Ddruid.indexer.runner.type=remote
-cp /usr/local/druid/lib/*
-cp /shared/docker/lib/*
io.druid.cli.Main server overlord
redirect_stderr=true
priority=100

2
integration-tests/docker/router.conf
View File

@ -10,7 +10,7 @@ command=java
-Ddruid.host=%(ENV_HOST_IP)s
-Ddruid.zk.service.host=druid-zookeeper-kafka
-Ddruid.server.http.numThreads=100
-cp /usr/local/druid/lib/*
-cp /shared/docker/lib/*
io.druid.cli.Main server router
redirect_stderr=true
priority=100

16
integration-tests/run_cluster.sh
View File

@ -28,25 +28,25 @@ mvn -B dependency:copy-dependencies -DoutputDirectory=$SHARED_DIR/docker/lib
docker build -t druid/cluster $SHARED_DIR/docker
# Start zookeeper and kafka
docker run -d --name druid-zookeeper-kafka -p 2181:2181 -p 9092:9092 -v $SHARED_DIR:/shared -v $DOCKERDIR/zookeeper.conf:$SUPERVISORDIR/zookeeper.conf -v $DOCKERDIR/kafka.conf:$SUPERVISORDIR/kafka.conf druid/cluster
docker run -d --privileged --name druid-zookeeper-kafka -p 2181:2181 -p 9092:9092 -v $SHARED_DIR:/shared -v $DOCKERDIR/zookeeper.conf:$SUPERVISORDIR/zookeeper.conf -v $DOCKERDIR/kafka.conf:$SUPERVISORDIR/kafka.conf druid/cluster
# Start MYSQL
docker run -d --name druid-metadata-storage -v $SHARED_DIR:/shared -v $DOCKERDIR/metadata-storage.conf:$SUPERVISORDIR/metadata-storage.conf druid/cluster
docker run -d --privileged --name druid-metadata-storage -v $SHARED_DIR:/shared -v $DOCKERDIR/metadata-storage.conf:$SUPERVISORDIR/metadata-storage.conf druid/cluster
# Start Overlord
docker run -d --name druid-overlord -p 8090:8090 -v $SHARED_DIR:/shared -v $DOCKERDIR/overlord.conf:$SUPERVISORDIR/overlord.conf --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
docker run -d --privileged --name druid-overlord -p 8090:8090 -v $SHARED_DIR:/shared -v $DOCKERDIR/overlord.conf:$SUPERVISORDIR/overlord.conf --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
# Start Coordinator
docker run -d --name druid-coordinator -p 8081:8081 -v $SHARED_DIR:/shared -v $DOCKERDIR/coordinator.conf:$SUPERVISORDIR/coordinator.conf --link druid-overlord:druid-overlord --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
docker run -d --privileged --name druid-coordinator -p 8081:8081 -v $SHARED_DIR:/shared -v $DOCKERDIR/coordinator.conf:$SUPERVISORDIR/coordinator.conf --link druid-overlord:druid-overlord --link druid-metadata-storage:druid-metadata-storage --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
# Start Historical
docker run -d --name druid-historical -v $SHARED_DIR:/shared -v $DOCKERDIR/historical.conf:$SUPERVISORDIR/historical.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
docker run -d --privileged --name druid-historical -v $SHARED_DIR:/shared -v $DOCKERDIR/historical.conf:$SUPERVISORDIR/historical.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka druid/cluster
# Start Middlemanger
docker run -d --name druid-middlemanager -p 8100:8100 -p 8101:8101 -p 8102:8102 -p 8103:8103 -p 8104:8104 -p 8105:8105 -v $RESOURCEDIR:/resources -v $SHARED_DIR:/shared -v $DOCKERDIR/middlemanager.conf:$SUPERVISORDIR/middlemanager.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-overlord:druid-overlord druid/cluster
docker run -d --privileged --name druid-middlemanager -p 8091:8091 -p 8100:8100 -p 8101:8101 -p 8102:8102 -p 8103:8103 -p 8104:8104 -p 8105:8105 -v $RESOURCEDIR:/resources -v $SHARED_DIR:/shared -v $DOCKERDIR/middlemanager.conf:$SUPERVISORDIR/middlemanager.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-overlord:druid-overlord druid/cluster
# Start Broker
docker run -d --name druid-broker -p 8082:8082 -v $SHARED_DIR:/shared -v $DOCKERDIR/broker.conf:$SUPERVISORDIR/broker.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-middlemanager:druid-middlemanager --link druid-historical:druid-historical druid/cluster
docker run -d --privileged --name druid-broker -p 8082:8082 -v $SHARED_DIR:/shared -v $DOCKERDIR/broker.conf:$SUPERVISORDIR/broker.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-middlemanager:druid-middlemanager --link druid-historical:druid-historical druid/cluster
# Start Router
docker run -d --name druid-router -p 8888:8888 -v $SHARED_DIR:/shared -v $DOCKERDIR/router.conf:$SUPERVISORDIR/router.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster
docker run -d --privileged --name druid-router -p 8888:8888 -v $SHARED_DIR:/shared -v $DOCKERDIR/router.conf:$SUPERVISORDIR/router.conf --link druid-zookeeper-kafka:druid-zookeeper-kafka --link druid-coordinator:druid-coordinator --link druid-broker:druid-broker druid/cluster

2
server/src/main/java/io/druid/client/selector/AbstractTierSelectorStrategy.java
View File

@ -23,6 +23,7 @@ import com.google.common.collect.Iterables;
import io.druid.timeline.DataSegment;
import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@ -38,6 +39,7 @@ public abstract class AbstractTierSelectorStrategy implements TierSelectorStrate
this.serverSelectorStrategy = serverSelectorStrategy;
}
@Nullable
@Override
public QueryableDruidServer pick(
Int2ObjectRBTreeMap<Set<QueryableDruidServer>> prioritizedServers,

3
server/src/main/java/io/druid/client/selector/DiscoverySelector.java
View File

@ -19,9 +19,12 @@
package io.druid.client.selector;
import javax.annotation.Nullable;
/**
*/
public interface DiscoverySelector<T>
{
@Nullable
T pick();
}

2
server/src/main/java/io/druid/client/selector/ServerSelector.java
View File

@ -23,6 +23,7 @@ import io.druid.server.coordination.DruidServerMetadata;
import io.druid.timeline.DataSegment;
import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap;
import javax.annotation.Nullable;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
@ -110,6 +111,7 @@ public class ServerSelector implements DiscoverySelector<QueryableDruidServer>
}
}
@Nullable
@Override
public QueryableDruidServer pick()
{

2
server/src/main/java/io/druid/client/selector/TierSelectorStrategy.java
View File

@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
import io.druid.timeline.DataSegment;
import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap;
import javax.annotation.Nullable;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
@ -40,6 +41,7 @@ public interface TierSelectorStrategy
{
Comparator<Integer> getComparator();
@Nullable
QueryableDruidServer pick(Int2ObjectRBTreeMap<Set<QueryableDruidServer>> prioritizedServers, DataSegment segment);
List<QueryableDruidServer> pick(

4
server/src/main/java/io/druid/curator/discovery/ServerDiscoveryFactory.java
View File

@ -46,14 +46,14 @@ public class ServerDiscoveryFactory
public ServerDiscoverySelector createSelector(String serviceName)
{
if (serviceName == null) {
return new ServerDiscoverySelector(new NoopServiceProvider());
return new ServerDiscoverySelector(new NoopServiceProvider(), serviceName);
}
final ServiceProvider serviceProvider = serviceDiscovery
.serviceProviderBuilder()
.serviceName(CuratorServiceUtils.makeCanonicalServiceName(serviceName))
.build();
return new ServerDiscoverySelector(serviceProvider);
return new ServerDiscoverySelector(serviceProvider, serviceName);
}
private static class NoopServiceProvider<T> implements ServiceProvider<T>

10
server/src/main/java/io/druid/curator/discovery/ServerDiscoverySelector.java
View File

@ -31,6 +31,7 @@ import io.druid.java.util.common.logger.Logger;
import org.apache.curator.x.discovery.ServiceInstance;
import org.apache.curator.x.discovery.ServiceProvider;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
@ -44,10 +45,12 @@ public class ServerDiscoverySelector implements DiscoverySelector<Server>
private static final Logger log = new Logger(ServerDiscoverySelector.class);
private final ServiceProvider serviceProvider;
private final String name;
public ServerDiscoverySelector(ServiceProvider serviceProvider)
public ServerDiscoverySelector(ServiceProvider serviceProvider, String name)
{
this.serviceProvider = serviceProvider;
this.name = name;
}
private static final Function<ServiceInstance, Server> TO_SERVER = new Function<ServiceInstance, Server>()
@ -97,6 +100,7 @@ public class ServerDiscoverySelector implements DiscoverySelector<Server>
}
};
@Nullable
@Override
public Server pick()
{
@ -105,12 +109,12 @@ public class ServerDiscoverySelector implements DiscoverySelector<Server>
instance = serviceProvider.getInstance();
}
catch (Exception e) {
log.info(e, "Exception getting instance");
log.info(e, "Exception getting instance for [%s]", name);
return null;
}
if (instance == null) {
log.error("No server instance found");
log.error("No server instance found for [%s]", name);
return null;
}

2
server/src/test/java/io/druid/curator/discovery/ServerDiscoverySelectorTest.java
View File

@ -46,7 +46,7 @@ public class ServerDiscoverySelectorTest
{
serviceProvider = EasyMock.createMock(ServiceProvider.class);
instance = EasyMock.createMock(ServiceInstance.class);
serverDiscoverySelector = new ServerDiscoverySelector(serviceProvider);
serverDiscoverySelector = new ServerDiscoverySelector(serviceProvider, "test");
}
@Test