From f9db3a1fc674e242c647d3ed6677859a4333334e Mon Sep 17 00:00:00 2001 From: fjy Date: Fri, 21 Nov 2014 15:46:14 -0800 Subject: [PATCH 1/3] fix crazy amounts of logging when coordinator loses leadership --- .../helper/DruidCoordinatorRuleRunner.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java index 03869810e85..dfe62334eeb 100644 --- a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java +++ b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java @@ -19,6 +19,8 @@ package io.druid.server.coordinator.helper; +import com.google.api.client.util.Maps; +import com.google.api.client.util.Sets; import com.metamx.emitter.EmittingLogger; import io.druid.metadata.MetadataRuleManager; import io.druid.server.coordinator.CoordinatorStats; @@ -31,6 +33,8 @@ import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import java.util.List; +import java.util.Map; +import java.util.Set; /** */ @@ -87,6 +91,8 @@ public class DruidCoordinatorRuleRunner implements DruidCoordinatorHelper // Run through all matched rules for available segments DateTime now = new DateTime(); MetadataRuleManager databaseRuleManager = paramsWithReplicationManager.getDatabaseRuleManager(); + + final Map> missingRules = Maps.newHashMap(); for (DataSegment segment : paramsWithReplicationManager.getAvailableSegments()) { List rules = databaseRuleManager.getRulesWithDefault(segment.getDataSource()); boolean foundMatchingRule = false; @@ -99,13 +105,21 @@ public class DruidCoordinatorRuleRunner implements DruidCoordinatorHelper } if (!foundMatchingRule) { - log.makeAlert("Unable to find a matching rule!") - .addData("dataSource", segment.getDataSource()) - .addData("segment", segment.getIdentifier()) - .emit(); + Set missingSegments = missingRules.get(segment.getDataSource()); + if (missingSegments == null) { + missingSegments = Sets.newHashSet(); + missingRules.put(segment.getDataSource(), missingSegments); + } + missingSegments.add(segment.getIdentifier()); } } + if (!missingRules.isEmpty()) { + log.makeAlert("Unable to find a matching rules!") + .addData("missingSegments", missingRules) + .emit(); + } + return paramsWithReplicationManager.buildFromExisting() .withCoordinatorStats(stats) .build(); From 10d45ab9cf4925fefb7b03ffee76714c42db8af1 Mon Sep 17 00:00:00 2001 From: fjy Date: Fri, 21 Nov 2014 16:48:45 -0800 Subject: [PATCH 2/3] address code review --- .../helper/DruidCoordinatorRuleRunner.java | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java index dfe62334eeb..387ce89d09b 100644 --- a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java +++ b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java @@ -19,8 +19,7 @@ package io.druid.server.coordinator.helper; -import com.google.api.client.util.Maps; -import com.google.api.client.util.Sets; +import com.google.common.collect.Lists; import com.metamx.emitter.EmittingLogger; import io.druid.metadata.MetadataRuleManager; import io.druid.server.coordinator.CoordinatorStats; @@ -33,14 +32,13 @@ import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import java.util.List; -import java.util.Map; -import java.util.Set; /** */ public class DruidCoordinatorRuleRunner implements DruidCoordinatorHelper { private static final EmittingLogger log = new EmittingLogger(DruidCoordinatorRuleRunner.class); + private static int MAX_MISSING_RULES = 10; private final ReplicationThrottler replicatorThrottler; @@ -92,7 +90,8 @@ public class DruidCoordinatorRuleRunner implements DruidCoordinatorHelper DateTime now = new DateTime(); MetadataRuleManager databaseRuleManager = paramsWithReplicationManager.getDatabaseRuleManager(); - final Map> missingRules = Maps.newHashMap(); + final List segmentsWithMissingRules = Lists.newArrayListWithCapacity(10); + int missingRules = 0; for (DataSegment segment : paramsWithReplicationManager.getAvailableSegments()) { List rules = databaseRuleManager.getRulesWithDefault(segment.getDataSource()); boolean foundMatchingRule = false; @@ -105,18 +104,17 @@ public class DruidCoordinatorRuleRunner implements DruidCoordinatorHelper } if (!foundMatchingRule) { - Set missingSegments = missingRules.get(segment.getDataSource()); - if (missingSegments == null) { - missingSegments = Sets.newHashSet(); - missingRules.put(segment.getDataSource(), missingSegments); + if (segmentsWithMissingRules.size() < MAX_MISSING_RULES) { + segmentsWithMissingRules.add(segment.getIdentifier()); } - missingSegments.add(segment.getIdentifier()); + missingRules++; } } - if (!missingRules.isEmpty()) { - log.makeAlert("Unable to find a matching rules!") - .addData("missingSegments", missingRules) + if (!segmentsWithMissingRules.isEmpty()) { + log.makeAlert("Unable to find matching rules!") + .addData("segmentsWithMissingRulesCount", missingRules) + .addData("segmentsWithMissingRules", segmentsWithMissingRules) .emit(); } From f518c530081c30750a246cab51507cd8223e6f9f Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 24 Nov 2014 10:12:29 -0800 Subject: [PATCH 3/3] address cr --- .../server/coordinator/helper/DruidCoordinatorRuleRunner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java index 387ce89d09b..be17219faf7 100644 --- a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java +++ b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java @@ -90,7 +90,7 @@ public class DruidCoordinatorRuleRunner implements DruidCoordinatorHelper DateTime now = new DateTime(); MetadataRuleManager databaseRuleManager = paramsWithReplicationManager.getDatabaseRuleManager(); - final List segmentsWithMissingRules = Lists.newArrayListWithCapacity(10); + final List segmentsWithMissingRules = Lists.newArrayListWithCapacity(MAX_MISSING_RULES); int missingRules = 0; for (DataSegment segment : paramsWithReplicationManager.getAvailableSegments()) { List rules = databaseRuleManager.getRulesWithDefault(segment.getDataSource());