HDDS-1353 : Metrics scm_pipeline_metrics_num_pipeline_creation_failed keeps increasing because of BackgroundPipelineCreator. (#681)

This commit is contained in:
avijayanhwx 2019-04-04 06:48:32 -07:00 committed by Nanda kumar
parent d2637cb176
commit 993f36ee33
4 changed files with 100 additions and 1 deletions

View File

@ -0,0 +1,36 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.pipeline;
import java.io.IOException;
/**
* Exception thrown when there are not enough Datanodes to create a pipeline.
*/
public class InsufficientDatanodesException extends IOException {
public InsufficientDatanodesException() {
super();
}
public InsufficientDatanodesException(String message) {
super(message);
}
}

View File

@ -105,7 +105,7 @@ public class RatisPipelineProvider implements PipelineProvider {
String e = String
.format("Cannot create pipeline of factor %d using %d nodes.",
factor.getNumber(), dns.size());
throw new IOException(e);
throw new InsufficientDatanodesException(e);
}
Pipeline pipeline = Pipeline.newBuilder()

View File

@ -152,6 +152,8 @@ public class SCMPipelineManager implements PipelineManager {
nodeManager.addPipeline(pipeline);
metrics.incNumPipelineCreated();
return pipeline;
} catch (InsufficientDatanodesException idEx) {
throw idEx;
} catch (IOException ex) {
metrics.incNumPipelineCreationFailed();
throw ex;

View File

@ -18,6 +18,9 @@
package org.apache.hadoop.hdds.scm.pipeline;
import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
@ -30,6 +33,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass;
import org.junit.Assert;
@ -208,4 +212,61 @@ public class TestSCMPipelineManager {
// clean up
pipelineManager.close();
}
@Test
public void testPipelineCreationFailedMetric() throws Exception {
MockNodeManager nodeManagerMock = new MockNodeManager(true,
20);
SCMPipelineManager pipelineManager =
new SCMPipelineManager(conf, nodeManagerMock, new EventQueue());
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManagerMock,
pipelineManager.getStateManager(), conf);
pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
mockRatisProvider);
MetricsRecordBuilder metrics = getMetrics(
SCMPipelineMetrics.class.getSimpleName());
long numPipelineCreated = getLongCounter("NumPipelineCreated",
metrics);
Assert.assertTrue(numPipelineCreated == 0);
// 3 DNs are unhealthy.
// Create 5 pipelines (Use up 15 Datanodes)
for (int i = 0; i < 5; i++) {
Pipeline pipeline = pipelineManager
.createPipeline(HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.THREE);
Assert.assertNotNull(pipeline);
}
metrics = getMetrics(
SCMPipelineMetrics.class.getSimpleName());
numPipelineCreated = getLongCounter("NumPipelineCreated", metrics);
Assert.assertTrue(numPipelineCreated == 5);
long numPipelineCreateFailed = getLongCounter(
"NumPipelineCreationFailed", metrics);
Assert.assertTrue(numPipelineCreateFailed == 0);
//This should fail...
try {
pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.THREE);
Assert.fail();
} catch (InsufficientDatanodesException idEx) {
Assert.assertEquals(
"Cannot create pipeline of factor 3 using 1 nodes.",
idEx.getMessage());
}
metrics = getMetrics(
SCMPipelineMetrics.class.getSimpleName());
numPipelineCreated = getLongCounter("NumPipelineCreated", metrics);
Assert.assertTrue(numPipelineCreated == 5);
numPipelineCreateFailed = getLongCounter(
"NumPipelineCreationFailed", metrics);
Assert.assertTrue(numPipelineCreateFailed == 0);
}
}