From 0d7986b76a479ff8758438a1fc409ca9343499d3 Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 2 Apr 2019 10:43:47 -0700 Subject: [PATCH 1/3] HDDS-1353 : Metrics scm_pipeline_metrics_num_pipeline_creation_failed keeps increasing because of BackgroundPipelineCreator. --- .../InsufficientDatanodesException.java | 36 +++++++++++ .../scm/pipeline/RatisPipelineProvider.java | 2 +- .../hdds/scm/pipeline/SCMPipelineManager.java | 2 + .../scm/pipeline/TestSCMPipelineManager.java | 60 +++++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java new file mode 100644 index 00000000000000..a6a5a69a165634 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InsufficientDatanodesException.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.pipeline; + +import java.io.IOException; + +/** + * Exception thrown when there are not enough Datanodes to create a pipeline. + */ +public class InsufficientDatanodesException extends IOException { + + + public InsufficientDatanodesException() { + super(); + } + + public InsufficientDatanodesException(String message) { + super(message); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java index 695220029a8e5e..6563e3fcaa4141 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java @@ -105,7 +105,7 @@ public Pipeline create(ReplicationFactor factor) throws IOException { String e = String .format("Cannot create pipeline of factor %d using %d nodes.", factor.getNumber(), dns.size()); - throw new IOException(e); + throw new InsufficientDatanodesException(e); } Pipeline pipeline = Pipeline.newBuilder() diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java index f274829df37d41..c72a52886c8256 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java @@ -152,6 +152,8 @@ public synchronized Pipeline createPipeline( nodeManager.addPipeline(pipeline); metrics.incNumPipelineCreated(); return pipeline; + } catch (InsufficientDatanodesException idEx) { + throw idEx; } catch (IOException ex) { metrics.incNumPipelineCreationFailed(); throw ex; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java index 990d73ab90f528..a3913e9f6dae59 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hdds.scm.pipeline; +import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; @@ -30,6 +33,7 @@ import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; import org.junit.Assert; @@ -208,4 +212,60 @@ public void testPipelineReport() throws IOException { // clean up pipelineManager.close(); } + + @Test + public void testPipelineCreationFailedMetric() throws Exception { + SCMPipelineManager pipelineManager = + new SCMPipelineManager(conf, nodeManager, new EventQueue()); + PipelineProvider mockRatisProvider = + new MockRatisPipelineProvider(nodeManager, + pipelineManager.getStateManager(), conf); + pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, + mockRatisProvider); + + MetricsRecordBuilder metrics = getMetrics( + SCMPipelineMetrics.class.getSimpleName()); + long numPipelineCreated = getLongCounter("NumPipelineCreated", + metrics); + Assert.assertTrue(numPipelineCreated == 0); + + // 3 DNs are unhealthy. + // Create 5 pipelines (Use up 15 Datanodes) + for (int i = 0; i < 5; i++) { + Pipeline pipeline = pipelineManager + .createPipeline(HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.THREE); + Assert.assertNotNull(pipeline); + } + + metrics = getMetrics( + SCMPipelineMetrics.class.getSimpleName()); + numPipelineCreated = getLongCounter("NumPipelineCreated", metrics); + Assert.assertTrue(numPipelineCreated == 5); + + long numPipelineCreateFailed = getLongCounter( + "NumPipelineCreationFailed", metrics); + Assert.assertTrue(numPipelineCreateFailed == 0); + + //This should fail... + try { + pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.THREE); + Assert.fail(); + } catch (InsufficientDatanodesException idEx) { + Assert.assertEquals( + "Cannot create pipeline of factor 3 using 1 nodes.", + idEx.getMessage()); + } + + metrics = getMetrics( + SCMPipelineMetrics.class.getSimpleName()); + numPipelineCreated = getLongCounter("NumPipelineCreated", metrics); + Assert.assertTrue(numPipelineCreated == 5); + + numPipelineCreateFailed = getLongCounter( + "NumPipelineCreationFailed", metrics); + Assert.assertTrue(numPipelineCreateFailed == 0); + } + } From e5f30bc006a1fd180c51073a308c6cf18f34ee29 Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 2 Apr 2019 13:57:41 -0700 Subject: [PATCH 2/3] HDDS-1353 : Metrics scm_pipeline_metrics_num_pipeline_creation_failed keeps increasing because of BackgroundPipelineCreator. --- .../apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java index a3913e9f6dae59..44cae54ff0f866 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java @@ -267,5 +267,4 @@ public void testPipelineCreationFailedMetric() throws Exception { "NumPipelineCreationFailed", metrics); Assert.assertTrue(numPipelineCreateFailed == 0); } - } From c6b13f6e1d9221847b860e79a8d91ad443b227e0 Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 2 Apr 2019 15:56:27 -0700 Subject: [PATCH 3/3] HDDS-1353 : Metrics scm_pipeline_metrics_num_pipeline_creation_failed keeps increasing because of BackgroundPipelineCreator. (Commit 3) --- .../hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java index 44cae54ff0f866..53e968bf23e6f8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java @@ -215,10 +215,12 @@ public void testPipelineReport() throws IOException { @Test public void testPipelineCreationFailedMetric() throws Exception { + MockNodeManager nodeManagerMock = new MockNodeManager(true, + 20); SCMPipelineManager pipelineManager = - new SCMPipelineManager(conf, nodeManager, new EventQueue()); + new SCMPipelineManager(conf, nodeManagerMock, new EventQueue()); PipelineProvider mockRatisProvider = - new MockRatisPipelineProvider(nodeManager, + new MockRatisPipelineProvider(nodeManagerMock, pipelineManager.getStateManager(), conf); pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider);