alert when no rules match; exceptions in rules no longer block

This commit is contained in:
Fangjin Yang 2012-12-14 09:05:25 -08:00
parent 6db99c45a6
commit aa69ed5251
4 changed files with 56 additions and 13 deletions

View File

@ -44,6 +44,7 @@ import com.metamx.druid.client.ServerInventoryManager;
import com.metamx.druid.coordination.DruidClusterInfo; import com.metamx.druid.coordination.DruidClusterInfo;
import com.metamx.druid.db.DatabaseRuleManager; import com.metamx.druid.db.DatabaseRuleManager;
import com.metamx.druid.db.DatabaseSegmentManager; import com.metamx.druid.db.DatabaseSegmentManager;
import com.metamx.emitter.EmittingLogger;
import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceEmitter;
import com.metamx.emitter.service.ServiceMetricEvent; import com.metamx.emitter.service.ServiceMetricEvent;
import com.metamx.phonebook.PhoneBook; import com.metamx.phonebook.PhoneBook;
@ -69,7 +70,7 @@ public class DruidMaster
{ {
public static final String MASTER_OWNER_NODE = "_MASTER"; public static final String MASTER_OWNER_NODE = "_MASTER";
private static final Logger log = new Logger(DruidMaster.class); private static final EmittingLogger log = new EmittingLogger(DruidMaster.class);
private final Object lock = new Object(); private final Object lock = new Object();
@ -575,7 +576,7 @@ public class DruidMaster
} }
} }
catch (Exception e) { catch (Exception e) {
log.error(e, "Caught exception, ignoring so that schedule keeps going."); log.makeAlert(e, "Caught exception, ignoring so that schedule keeps going.").emit();
} }
} }
} }

View File

@ -25,6 +25,7 @@ import com.metamx.druid.client.DataSegment;
import com.metamx.druid.db.DatabaseRuleManager; import com.metamx.druid.db.DatabaseRuleManager;
import com.metamx.druid.master.rules.Rule; import com.metamx.druid.master.rules.Rule;
import com.metamx.druid.master.rules.RuleMap; import com.metamx.druid.master.rules.RuleMap;
import com.metamx.emitter.EmittingLogger;
import java.util.List; import java.util.List;
@ -32,7 +33,7 @@ import java.util.List;
*/ */
public class DruidMasterRuleRunner implements DruidMasterHelper public class DruidMasterRuleRunner implements DruidMasterHelper
{ {
private static final Logger log = new Logger(DruidMasterRuleRunner.class); private static final EmittingLogger log = new EmittingLogger(DruidMasterRuleRunner.class);
private final DruidMaster master; private final DruidMaster master;
@ -67,7 +68,7 @@ public class DruidMasterRuleRunner implements DruidMasterHelper
} }
if (!foundMatchingRule) { if (!foundMatchingRule) {
throw new ISE("Unable to find a matching rule for segment[%s]", segment.getIdentifier()); log.makeAlert("Unable to find a matching rule for segment[%s]", segment.getIdentifier()).emit();
} }
} }

View File

@ -52,7 +52,7 @@ public abstract class LoadRule implements Rule
MinMaxPriorityQueue<ServerHolder> serverQueue = params.getDruidCluster().getServersByTier(getTier()); MinMaxPriorityQueue<ServerHolder> serverQueue = params.getDruidCluster().getServersByTier(getTier());
if (serverQueue == null) { if (serverQueue == null) {
log.makeAlert("Tier[%s] has no servers! Check your cluster configuration!", getTier()).emit(); log.makeAlert("Tier[%s] has no servers! Check your cluster configuration!", getTier()).emit();
throw new ISE("Tier[%s] has no servers! Check your cluster configuration!", getTier()); return stats;
} }
stats.accumulate(assign(expectedReplicants, actualReplicants, serverQueue, segment)); stats.accumulate(assign(expectedReplicants, actualReplicants, serverQueue, segment));

View File

@ -397,20 +397,61 @@ public class DruidMasterRuleRunnerTest
.withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster())) .withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster()))
.build(); .build();
boolean exceptionOccurred = false;
try {
ruleRunner.run(params);
}
catch (Exception e) {
exceptionOccurred = true;
}
Assert.assertTrue(exceptionOccurred); ruleRunner.run(params);
EasyMock.verify(emitter); EasyMock.verify(emitter);
EasyMock.verify(mockPeon); EasyMock.verify(mockPeon);
} }
@Test
public void testRunRuleDoesNotExist() throws Exception
{
emitter.emit(EasyMock.<ServiceEventBuilder>anyObject());
EasyMock.expectLastCall().atLeastOnce();
EasyMock.replay(emitter);
EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.<String>anyObject())).andReturn(
Lists.<Rule>newArrayList(
new IntervalLoadRule(new Interval("2012-01-02T00:00:00.000Z/2012-01-03T00:00:00.000Z"), 1, "normal")
)
).atLeastOnce();
EasyMock.replay(databaseRuleManager);
DruidCluster druidCluster = new DruidCluster(
ImmutableMap.of(
"normal",
MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create(
Arrays.asList(
new ServerHolder(
new DruidServer(
"serverNorm",
"hostNorm",
1000,
"historical",
"normal"
),
mockPeon
)
)
)
)
);
DruidMasterRuntimeParams params =
new DruidMasterRuntimeParams.Builder()
.withEmitter(emitter)
.withDruidCluster(druidCluster)
.withAvailableSegments(availableSegments)
.withDatabaseRuleManager(databaseRuleManager)
.withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster()))
.build();
ruleRunner.run(params);
EasyMock.verify(emitter);
}
@Test @Test
public void testDropRemove() throws Exception public void testDropRemove() throws Exception
{ {