From c863b51baae95784c0560e07d6ab0a67659a0314 Mon Sep 17 00:00:00 2001 From: Zhihong Yu Date: Tue, 17 Jul 2012 22:02:06 +0000 Subject: [PATCH] HBASE-6405 Create Hadoop compatibilty modules and Metrics2 implementation of replication metrics (Elliot Clark) git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1362681 13f79535-47bb-0310-9956-ffa450edef68 --- hbase-hadoop-compat/pom.xml | 63 ++++++ .../hbase/metrics/BaseMetricsSource.java | 72 +++++++ .../metrics/ReplicationMetricsSource.java | 30 +++ .../ReplicationMetricsSourceFactory.java | 55 +++++ .../ReplicationMetricsSourceFactoryTest.java | 34 +++ hbase-hadoop1-compat/pom.xml | 103 +++++++++ .../hbase/metrics/BaseMetricsSourceImpl.java | 201 ++++++++++++++++++ .../metrics/ReplicationMetricsSourceImpl.java | 43 ++++ ...ionserver.metrics.ReplicationMetricsSource | 1 + .../metrics/BaseMetricsSourceImplTest.java | 97 +++++++++ .../ReplicationMetricsSourceImplTest.java | 35 +++ hbase-hadoop2-compat/pom.xml | 105 +++++++++ .../hbase/metrics/BaseMetricsSourceImpl.java | 193 +++++++++++++++++ .../metrics/ReplicationMetricsSourceImpl.java | 43 ++++ .../metrics2/lib/HBaseMetricsFactory.java | 50 +++++ ...ionserver.metrics.ReplicationMetricsSource | 1 + .../metrics/BaseMetricsSourceImplTest.java | 88 ++++++++ .../ReplicationMetricsSourceImplTest.java | 33 +++ hbase-server/pom.xml | 9 + .../regionserver/ReplicationSink.java | 18 +- .../regionserver/ReplicationSinkMetrics.java | 82 ------- .../regionserver/ReplicationSource.java | 15 +- .../ReplicationSourceMetrics.java | 125 ----------- .../regionserver/ReplicationStatistics.java | 47 ---- .../metrics/ReplicationSinkMetrics.java | 62 ++++++ .../metrics/ReplicationSourceMetrics.java | 157 ++++++++++++++ pom.xml | 32 ++- .../{all.xml => hadoop-one-compat.xml} | 5 +- src/assembly/hadoop-two-compat.xml | 156 ++++++++++++++ 29 files changed, 1678 insertions(+), 277 deletions(-) create mode 100644 hbase-hadoop-compat/pom.xml create mode 100644 hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSource.java create mode 100644 hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSource.java create mode 100644 hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceFactory.java create mode 100644 hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceFactoryTest.java create mode 100644 hbase-hadoop1-compat/pom.xml create mode 100644 hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java create mode 100644 hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImpl.java create mode 100644 hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSource create mode 100644 hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImplTest.java create mode 100644 hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImplTest.java create mode 100644 hbase-hadoop2-compat/pom.xml create mode 100644 hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java create mode 100644 hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImpl.java create mode 100644 hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/HBaseMetricsFactory.java create mode 100644 hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSource create mode 100644 hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImplTest.java create mode 100644 hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImplTest.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkMetrics.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSinkMetrics.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSourceMetrics.java rename src/assembly/{all.xml => hadoop-one-compat.xml} (95%) create mode 100644 src/assembly/hadoop-two-compat.xml diff --git a/hbase-hadoop-compat/pom.xml b/hbase-hadoop-compat/pom.xml new file mode 100644 index 00000000000..05b06fc49dc --- /dev/null +++ b/hbase-hadoop-compat/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + + hbase + org.apache.hbase + 0.95-SNAPSHOT + .. + + + hbase-hadoop-compat + HBase - Hadoop Compatibility + + Interfaces to be implemented in order to smooth + over hadoop version differences + + + + + + maven-surefire-plugin + + + + secondPartTestsExecution + test + + test + + + true + + + + + + + + + + + diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSource.java new file mode 100644 index 00000000000..23130ea4fd0 --- /dev/null +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSource.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics; + +/** + * BaseMetricsSource for dynamic metrics to announce to Metrics2 + */ +public interface BaseMetricsSource { + + /** + * Set a gauge to a specific value. + * + * @param gaugeName the name of the gauge + * @param value the value + */ + public void setGauge(String gaugeName, long value); + + /** + * Add some amount to a gauge. + * + * @param gaugeName the name of the gauge + * @param delta the amount to change the gauge by. + */ + public void incGauge(String gaugeName, long delta); + + /** + * Subtract some amount from a gauge. + * + * @param gaugeName the name of the gauge + * @param delta the amount to change the gauge by. + */ + public void decGauge(String gaugeName, long delta); + + /** + * Remove a gauge and no longer announce it. + * + * @param key Name of the gauge to remove. + */ + public void removeGauge(String key); + + /** + * Add some amount to a counter. + * + * @param counterName the name of the counter + * @param delta the amount to change the counter by. + */ + public void incCounters(String counterName, long delta); + + /** + * Remove a counter and stop announcing it to metrics2. + * + * @param key + */ + public void removeCounter(String key); + +} diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSource.java new file mode 100644 index 00000000000..2e4d28d4588 --- /dev/null +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSource.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication.regionserver.metrics; + +import org.apache.hadoop.hbase.metrics.BaseMetricsSource; + +/** + * Provides access to gauges and counters. Implementers will hide the details of hadoop1 or + * hadoop2's metrics2 classes and publishing. + */ +public interface ReplicationMetricsSource extends BaseMetricsSource { + //Empty interface so that ServiceLoader can find the right implementation. + +} diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceFactory.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceFactory.java new file mode 100644 index 00000000000..83a829f4d94 --- /dev/null +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceFactory.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication.regionserver.metrics; + +import java.util.ServiceLoader; + +/** + * Class to load ReplicationMetricsSource from the class path. Will only return a singleton + * instance. + */ +public class ReplicationMetricsSourceFactory { + + private static ReplicationMetricsSource rms = null; + public static final String EXCEPTION_STRING = "Could not create a Replication metrics source. " + + "Is the hadoop compatibility jar on the classpath?"; + + /** + * Get the singleton instance of ReplicationMetricsSource + * + * @return the singleton + */ + public static synchronized ReplicationMetricsSource getInstance() { + if (rms == null) { + try { + rms = ServiceLoader.load(ReplicationMetricsSource.class).iterator().next(); + } catch (Exception e) { + throw new RuntimeException(EXCEPTION_STRING, e); + } catch (Error e) { + throw new RuntimeException(EXCEPTION_STRING, e); + } + + // If there was nothing returned and no exception then throw an exception. + if (rms == null) { + throw new RuntimeException(EXCEPTION_STRING); + } + } + return rms; + } +} diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceFactoryTest.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceFactoryTest.java new file mode 100644 index 00000000000..d82e6282c77 --- /dev/null +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceFactoryTest.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication.regionserver.metrics; + +import org.junit.Test; + +/** + * Test for the ReplicationMetricsSourceFactory + */ +public class ReplicationMetricsSourceFactoryTest { + + @Test(expected=RuntimeException.class) + public void testGetInstanceNoHadoopCompat() throws Exception { + //This should throw an exception because there is no compat lib on the class path. + ReplicationMetricsSourceFactory.getInstance(); + + } +} diff --git a/hbase-hadoop1-compat/pom.xml b/hbase-hadoop1-compat/pom.xml new file mode 100644 index 00000000000..fcddcf24ead --- /dev/null +++ b/hbase-hadoop1-compat/pom.xml @@ -0,0 +1,103 @@ + + + 4.0.0 + + hbase + org.apache.hbase + 0.95-SNAPSHOT + .. + + + hbase-hadoop1-compat + HBase - Hadoop One Compatibility + + Interfaces to be implemented in order to smooth + over hadoop version differences + + + + + + maven-surefire-plugin + + + + secondPartTestsExecution + test + + test + + + true + + + + + + + + + + + org.apache.hbase + hbase-hadoop-compat + + + org.apache.hadoop + hadoop-core + ${hadoop-one.version} + true + + + hsqldb + hsqldb + + + net.sf.kosmosfs + kfs + + + org.eclipse.jdt + core + + + net.java.dev.jets3t + jets3t + + + oro + oro + + + + + org.apache.hadoop + hadoop-test + ${hadoop-one.version} + true + test + + + + diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java new file mode 100644 index 00000000000..189a6a5722c --- /dev/null +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.metrics2.MetricsBuilder; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricMutableCounterLong; +import org.apache.hadoop.metrics2.lib.MetricMutableGaugeLong; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Hadoop 1 implementation of BaseMetricsSource + */ +public class BaseMetricsSourceImpl implements BaseMetricsSource, MetricsSource { + + private static boolean defaultMetricsSystemInited = false; + public static final String HBASE_METRICS_SYSTEM_NAME = "hbase"; + + public ConcurrentMap + gauges = new ConcurrentHashMap(); + public ConcurrentMap counters = + new ConcurrentHashMap(); + + protected String metricsContext; + protected String metricsName; + protected String metricsDescription; + + public BaseMetricsSourceImpl( + String metricsName, + String metricsDescription, + String metricsContext) { + this.metricsContext = metricsContext; + this.metricsName = metricsName; + this.metricsDescription = metricsDescription; + + if (!defaultMetricsSystemInited) { + //Not too worried about mutli-threaded here as all it does is spam the logs. + defaultMetricsSystemInited = true; + DefaultMetricsSystem.initialize(HBASE_METRICS_SYSTEM_NAME); + } + + //Register this instance. + DefaultMetricsSystem.registerSource(this.metricsContext, this.metricsDescription, this); + } + + /** + * Set a single gauge to a value. + * + * @param gaugeName gauge name + * @param value the new value of the gauge. + */ + public void setGauge(String gaugeName, long value) { + MetricMutableGaugeLong gaugeInt = getLongGauge(gaugeName, value); + gaugeInt.set(value); + } + + /** + * Add some amount to a gauge. + * + * @param gaugeName The name of the gauge to increment. + * @param delta The amount to increment the gauge by. + */ + public void incGauge(String gaugeName, long delta) { + MetricMutableGaugeLong gaugeInt = getLongGauge(gaugeName, 0l); + gaugeInt.incr(delta); + } + + /** + * Decrease the value of a named gauge. + * + * @param gaugeName The name of the gauge. + * @param delta the ammount to subtract from a gauge value. + */ + public void decGauge(String gaugeName, long delta) { + MetricMutableGaugeLong gaugeInt = getLongGauge(gaugeName, 0l); + gaugeInt.decr(delta); + } + + /** + * Increment a named counter by some value. + * + * @param key the name of the counter + * @param delta the ammount to increment + */ + public void incCounters(String key, long delta) { + MetricMutableCounterLong counter = getLongCounter(key, 0l); + counter.incr(delta); + + } + + /** + * Remove a named gauge. + * + * @param key + */ + public void removeGauge(String key) { + gauges.remove(key); + } + + /** + * Remove a named counter. + * + * @param key + */ + public void removeCounter(String key) { + counters.remove(key); + } + + /** + * Method to export all the metrics. + * + * @param metricsBuilder Builder to accept metrics + * @param all push all or only changed? + */ + @Override + public void getMetrics(MetricsBuilder metricsBuilder, boolean all) { + + MetricsRecordBuilder rb = metricsBuilder.addRecord(metricsName).setContext(metricsContext); + + for (Map.Entry entry : counters.entrySet()) { + entry.getValue().snapshot(rb, all); + } + for (Map.Entry entry : gauges.entrySet()) { + entry.getValue().snapshot(rb, all); + } + + } + + /** + * Get a MetricMutableGaugeLong from the storage. If it is not there atomically put it. + * + * @param gaugeName name of the gauge to create or get. + * @param potentialStartingValue value of the new counter if we have to create it. + * @return + */ + private MetricMutableGaugeLong getLongGauge(String gaugeName, long potentialStartingValue) { + //Try and get the guage. + MetricMutableGaugeLong gauge = gauges.get(gaugeName); + + //If it's not there then try and put a new one in the storage. + if (gauge == null) { + + //Create the potential new gauge. + MetricMutableGaugeLong newGauge = new MetricMutableGaugeLong(gaugeName, "", + potentialStartingValue); + + // Try and put the gauge in. This is atomic. + gauge = gauges.putIfAbsent(gaugeName, newGauge); + + //If the value we get back is null then the put was successful and we will return that. + //otherwise gaugeLong should contain the thing that was in before the put could be completed. + if (gauge == null) { + gauge = newGauge; + } + } + return gauge; + } + + /** + * Get a MetricMutableCounterLong from the storage. If it is not there atomically put it. + * + * @param counterName Name of the counter to get + * @param potentialStartingValue starting value if we have to create a new counter + * @return + */ + private MetricMutableCounterLong getLongCounter(String counterName, long potentialStartingValue) { + //See getLongGauge for description on how this works. + MetricMutableCounterLong counter = counters.get(counterName); + if (counter == null) { + MetricMutableCounterLong newCounter = + new MetricMutableCounterLong(counterName, "", potentialStartingValue); + counter = counters.putIfAbsent(counterName, newCounter); + if (counter == null) { + counter = newCounter; + } + } + return counter; + } + +} diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImpl.java new file mode 100644 index 00000000000..c0fd8f6e258 --- /dev/null +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImpl.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication.regionserver.metrics; + +import org.apache.hadoop.hbase.metrics.BaseMetricsSourceImpl; +import org.apache.hadoop.metrics2.MetricsSource; + +/** + * Hadoop1 implementation of ReplicationMetricsSource. This provides access to metrics gauges and + * counters. + */ +public class ReplicationMetricsSourceImpl extends BaseMetricsSourceImpl implements + ReplicationMetricsSource { + + public static final String METRICS_NAME = "ReplicationMetrics"; + public static final String METRICS_CONTEXT = "replicationmetrics"; + public static final String METRICS_DESCRIPTION = "Metrics about HBase replication"; + + public ReplicationMetricsSourceImpl() { + this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT); + } + + ReplicationMetricsSourceImpl(String metricsName, + String metricsDescription, String metricsContext) { + super(metricsName, metricsDescription, metricsContext); + } +} diff --git a/hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSource b/hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSource new file mode 100644 index 00000000000..bb64ad5ba0d --- /dev/null +++ b/hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSource @@ -0,0 +1 @@ +org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSourceImpl \ No newline at end of file diff --git a/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImplTest.java b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImplTest.java new file mode 100644 index 00000000000..2a3dc276657 --- /dev/null +++ b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImplTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.metrics2.lib.MetricMutableCounterLong; +import org.apache.hadoop.metrics2.lib.MetricMutableGaugeLong; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; + +/** + * Test of the default BaseMetricsSource implementation for hadoop 1 + */ +public class BaseMetricsSourceImplTest { + + private static BaseMetricsSourceImpl bmsi; + + @BeforeClass + public static void setUp() throws Exception { + bmsi = new BaseMetricsSourceImpl("TestName", "test description", "testcontext"); + } + + @Test + public void testSetGauge() throws Exception { + String key = "testset"; + bmsi.setGauge(key, 100); + MetricMutableGaugeLong g = bmsi.gauges.get(key); + assertEquals(key, g.name); + bmsi.setGauge(key, 110); + assertSame(g, bmsi.gauges.get(key)); + + } + + @Test + public void testIncGauge() throws Exception { + String key = "testincgauge"; + bmsi.incGauge(key, 100); + MetricMutableGaugeLong g = bmsi.gauges.get(key); + assertEquals(key, g.name); + bmsi.incGauge(key, 10); + assertSame(g, bmsi.gauges.get(key)); + } + + @Test + public void testDecGauge() throws Exception { + String key = "testdec"; + bmsi.decGauge(key, 100); + MetricMutableGaugeLong g = bmsi.gauges.get(key); + assertEquals(key, g.name); + bmsi.decGauge(key, 100); + assertSame(g, bmsi.gauges.get(key)); + } + + @Test + public void testIncCounters() throws Exception { + String key = "testinccounter"; + bmsi.incCounters(key, 100); + MetricMutableCounterLong c = bmsi.counters.get(key); + assertEquals(key, c.name); + bmsi.incCounters(key, 100); + assertSame(c, bmsi.counters.get(key)); + } + + @Test + public void testRemoveGauge() throws Exception { + bmsi.setGauge("testrm", 100); + bmsi.removeGauge("testrm"); + assertNull(bmsi.gauges.get("testrm")); + + } + + @Test + public void testRemoveCounter() throws Exception { + bmsi.incCounters("testrm", 100); + bmsi.removeCounter("testrm"); + assertNull(bmsi.counters.get("testrm")); + } +} diff --git a/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImplTest.java b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImplTest.java new file mode 100644 index 00000000000..717aca57bab --- /dev/null +++ b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImplTest.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication.regionserver.metrics; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * Test to make sure that ReplicationMetricsSourceImpl is hooked up to ServiceLoader + */ +public class ReplicationMetricsSourceImplTest { + + @Test + public void testGetInstance() throws Exception { + ReplicationMetricsSource rms = ReplicationMetricsSourceFactory.getInstance(); + assertTrue(rms instanceof ReplicationMetricsSourceImpl); + } +} diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml new file mode 100644 index 00000000000..bed591cd4d5 --- /dev/null +++ b/hbase-hadoop2-compat/pom.xml @@ -0,0 +1,105 @@ + + + 4.0.0 + + hbase + org.apache.hbase + 0.95-SNAPSHOT + .. + + + hbase-hadoop2-compat + HBase - Hadoop Two Compatibility + + Interfaces to be implemented in order to smooth + over hadoop version differences + + + + + + maven-surefire-plugin + + + + secondPartTestsExecution + test + + test + + + true + + + + + + maven-dependency-plugin + + + create-mrapp-generated-classpath + generate-test-resources + + build-classpath + + + + + ${project.build.directory}/test-classes/mrapp-generated-classpath + + + + + + + + + + + + org.apache.hbase + hbase-hadoop-compat + + + org.apache.hadoop + hadoop-client + ${hadoop-two.version} + + + org.apache.hadoop + hadoop-annotations + ${hadoop-two.version} + + + + org.apache.hadoop + hadoop-minicluster + ${hadoop-two.version} + + + diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java new file mode 100644 index 00000000000..769b7e924a5 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.HBaseMetricsFactory; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableGaugeLong; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** Hadoop 2 implementation of BaseMetricsSource for */ +public class BaseMetricsSourceImpl implements BaseMetricsSource, MetricsSource { + + private static boolean defaultMetricsSystemInited = false; + public static final String HBASE_METRICS_SYSTEM_NAME = "hbase"; + + public ConcurrentMap + gauges = new ConcurrentHashMap(); + public ConcurrentMap counters = + new ConcurrentHashMap(); + + protected String metricsContext; + protected String metricsName; + protected String metricsDescription; + + public BaseMetricsSourceImpl(String metricsName, + String metricsDescription, + String metricsContext) { + this.metricsContext = metricsContext; + this.metricsName = metricsName; + this.metricsDescription = metricsDescription; + + if (!defaultMetricsSystemInited) { + //Not too worried about mutlithread here as all it does is spam the logs. + defaultMetricsSystemInited = true; + DefaultMetricsSystem.initialize(HBASE_METRICS_SYSTEM_NAME); + } + DefaultMetricsSystem.instance().register(this.metricsContext, this.metricsDescription, this); + + } + + /** + * Set a single gauge to a value. + * + * @param gaugeName gauge name + * @param value the new value of the gauge. + */ + public void setGauge(String gaugeName, long value) { + MutableGaugeLong gaugeInt = getLongGauge(gaugeName, value); + gaugeInt.set(value); + } + + /** + * Add some amount to a gauge. + * + * @param gaugeName The name of the gauge to increment. + * @param delta The amount to increment the gauge by. + */ + public void incGauge(String gaugeName, long delta) { + MutableGaugeLong gaugeInt = getLongGauge(gaugeName, 0l); + gaugeInt.incr(delta); + } + + /** + * Decrease the value of a named gauge. + * + * @param gaugeName The name of the gauge. + * @param delta the ammount to subtract from a gauge value. + */ + public void decGauge(String gaugeName, long delta) { + MutableGaugeLong gaugeInt = getLongGauge(gaugeName, 0l); + gaugeInt.decr(delta); + } + + /** + * Increment a named counter by some value. + * + * @param key the name of the counter + * @param delta the ammount to increment + */ + public void incCounters(String key, long delta) { + MutableCounterLong counter = getLongCounter(key, 0l); + counter.incr(delta); + + } + + /** + * Remove a named gauge. + * + * @param key + */ + public void removeGauge(String key) { + gauges.remove(key); + } + + /** + * Remove a named counter. + * + * @param key + */ + public void removeCounter(String key) { + counters.remove(key); + } + + @Override + public void getMetrics(MetricsCollector metricsCollector, boolean all) { + MetricsRecordBuilder rb = + metricsCollector.addRecord(this.metricsName).setContext(metricsContext); + + for (Map.Entry entry : counters.entrySet()) { + entry.getValue().snapshot(rb, all); + } + for (Map.Entry entry : gauges.entrySet()) { + entry.getValue().snapshot(rb, all); + } + + } + + /** + * Get a MetricMutableGaugeLong from the storage. If it is not there atomically put it. + * + * @param gaugeName name of the gauge to create or get. + * @param potentialStartingValue value of the new counter if we have to create it. + * @return + */ + private MutableGaugeLong getLongGauge(String gaugeName, long potentialStartingValue) { + //Try and get the guage. + MutableGaugeLong gaugeInt = gauges.get(gaugeName); + + //If it's not there then try and put a new one in the storage. + if (gaugeInt == null) { + + //Create the potential new gauge. + MutableGaugeLong newGauge = HBaseMetricsFactory.newGauge(gaugeName, + "", + potentialStartingValue); + + // Try and put the gauge in. This is atomic. + gaugeInt = gauges.putIfAbsent(gaugeName, newGauge); + + //If the value we get back is null then the put was successful and we will return that. + //otherwise gaugeInt should contain the thing that was in before the put could be completed. + if (gaugeInt == null) { + gaugeInt = newGauge; + } + } + return gaugeInt; + } + + /** + * Get a MetricMutableCounterLong from the storage. If it is not there atomically put it. + * + * @param counterName Name of the counter to get + * @param potentialStartingValue starting value if we have to create a new counter + * @return + */ + private MutableCounterLong getLongCounter(String counterName, long potentialStartingValue) { + //See getLongGauge for description on how this works. + MutableCounterLong counter = counters.get(counterName); + if (counter == null) { + MutableCounterLong newCounter = + HBaseMetricsFactory.newCounter(counterName, "", potentialStartingValue); + counter = counters.putIfAbsent(counterName, newCounter); + if (counter == null) { + counter = newCounter; + } + } + return counter; + } + +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImpl.java new file mode 100644 index 00000000000..002316eb472 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImpl.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication.regionserver.metrics; + +import org.apache.hadoop.hbase.metrics.BaseMetricsSourceImpl; +import org.apache.hadoop.metrics2.MetricsSource; + +/** + * Hadoop2 implementation of ReplicationMetricsSource. This provides access to metrics gauges and + * counters. + */ +public class ReplicationMetricsSourceImpl extends BaseMetricsSourceImpl implements + ReplicationMetricsSource { + + public static final String METRICS_NAME = "ReplicationMetrics"; + public static final String METRICS_CONTEXT = "replicationmetrics"; + public static final String METRICS_DESCRIPTION = "Metrics about HBase replication"; + + public ReplicationMetricsSourceImpl() { + this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT); + } + + ReplicationMetricsSourceImpl(String metricsName, + String metricsDescription, String metricsContext) { + super(metricsName, metricsDescription, metricsContext); + } +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/HBaseMetricsFactory.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/HBaseMetricsFactory.java new file mode 100644 index 00000000000..8b14ff69b51 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/HBaseMetricsFactory.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +/** + * Factory providing static methods to create MutableMetrics classes. + * HBase uses this class rather than MetricsRegistry because MetricsRegistry does not + * allow metrics to be removed. + */ +public class HBaseMetricsFactory { + + /** + * Create a new gauge + * @param name Name of the gauge + * @param desc Description of the gauge + * @param startingValue The starting value + * @return a new MutableGaugeLong that has a starting value. + */ + public static MutableGaugeLong newGauge(String name, String desc, long startingValue) { + return new MutableGaugeLong(Interns.info(name, desc), startingValue); + } + + /** + * Create a new counter. + * @param name Name of the counter. + * @param desc Description of the counter. + * @param startingValue The starting value. + * @return a new MutableCounterLong that has a starting value. + */ + public static MutableCounterLong newCounter(String name, String desc, long startingValue) { + return new MutableCounterLong(Interns.info(name, desc), startingValue); + } + +} diff --git a/hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSource b/hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSource new file mode 100644 index 00000000000..bb64ad5ba0d --- /dev/null +++ b/hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSource @@ -0,0 +1 @@ +org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSourceImpl \ No newline at end of file diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImplTest.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImplTest.java new file mode 100644 index 00000000000..0c8046dfefe --- /dev/null +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImplTest.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics; + +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Test of default BaseMetricsSource for hadoop 2 + */ +public class BaseMetricsSourceImplTest { + + private static BaseMetricsSourceImpl bmsi; + + @BeforeClass + public static void setUp() throws Exception { + bmsi = new BaseMetricsSourceImpl("TestName", "test description", "testcontext"); + } + + @Test + public void testSetGauge() throws Exception { + bmsi.setGauge("testset", 100); + assertEquals(100, bmsi.gauges.get("testset").value()); + bmsi.setGauge("testset", 300); + assertEquals(300, bmsi.gauges.get("testset").value()); + + } + + @Test + public void testIncGauge() throws Exception { + bmsi.incGauge("testincgauge", 100); + assertEquals(100, bmsi.gauges.get("testincgauge").value()); + bmsi.incGauge("testincgauge", 100); + assertEquals(200, bmsi.gauges.get("testincgauge").value()); + + } + + @Test + public void testDecGauge() throws Exception { + bmsi.decGauge("testdec", 100); + assertEquals(-100, bmsi.gauges.get("testdec").value()); + bmsi.decGauge("testdec", 100); + assertEquals(-200, bmsi.gauges.get("testdec").value()); + + } + + @Test + public void testIncCounters() throws Exception { + bmsi.incCounters("testinccounter", 100); + assertEquals(100, bmsi.counters.get("testinccounter").value()); + bmsi.incCounters("testinccounter", 100); + assertEquals(200, bmsi.counters.get("testinccounter").value()); + + } + + @Test + public void testRemoveGauge() throws Exception { + bmsi.setGauge("testrmgauge", 100); + bmsi.removeGauge("testrmgauge"); + assertNull(bmsi.gauges.get("testrmgauge")); + } + + @Test + public void testRemoveCounter() throws Exception { + bmsi.incCounters("testrmcounter", 100); + bmsi.removeCounter("testrmcounter"); + assertNull(bmsi.counters.get("testrmcounter")); + } +} diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImplTest.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImplTest.java new file mode 100644 index 00000000000..af31c6ee666 --- /dev/null +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSourceImplTest.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication.regionserver.metrics; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** Test for ReplicationMetricsSourceImpl */ +public class ReplicationMetricsSourceImplTest { + + @Test + public void testGetInstance() throws Exception { + ReplicationMetricsSource rms = ReplicationMetricsSourceFactory.getInstance(); + assertTrue(rms instanceof ReplicationMetricsSourceImpl); + } +} diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 4359d9cf1e2..850aa03eec3 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -288,6 +288,15 @@ org.apache.hbase hbase-common + + org.apache.hbase + hbase-hadoop-compat + + + org.apache.hbase + ${compat.module} + ${project.version} + io.netty diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java index 0b6987ecdd3..a359f782a8a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java @@ -19,13 +19,6 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.Map.Entry; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -39,8 +32,16 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Row; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationSinkMetrics; import org.apache.hadoop.hbase.util.Bytes; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + /** * This class is responsible for replicating the edits coming * from another cluster. @@ -133,7 +134,7 @@ public class ReplicationSink { } this.metrics.setAgeOfLastAppliedOp( entries[entries.length-1].getKey().getWriteTime()); - this.metrics.appliedBatchesRate.inc(1); + this.metrics.applyBatch(entries.length); LOG.info("Total replicated: " + totalReplicated); } catch (IOException ex) { LOG.error("Unable to accept edit because:", ex); @@ -173,7 +174,6 @@ public class ReplicationSink { try { table = this.pool.getTable(tableName); table.batch(rows); - this.metrics.appliedOpsRate.inc(rows.size()); } catch (InterruptedException ix) { throw new IOException(ix); } finally { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkMetrics.java deleted file mode 100644 index bf324e2c94b..00000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkMetrics.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Copyright 2010 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.metrics.MetricsRate; -import org.apache.hadoop.metrics.MetricsContext; -import org.apache.hadoop.metrics.MetricsRecord; -import org.apache.hadoop.metrics.MetricsUtil; -import org.apache.hadoop.metrics.Updater; -import org.apache.hadoop.metrics.jvm.JvmMetrics; -import org.apache.hadoop.metrics.util.MetricsIntValue; -import org.apache.hadoop.metrics.util.MetricsLongValue; -import org.apache.hadoop.metrics.util.MetricsRegistry; - -/** - * This class is for maintaining the various replication statistics - * for a sink and publishing them through the metrics interfaces. - */ -@InterfaceAudience.Private -public class ReplicationSinkMetrics implements Updater { - private final MetricsRecord metricsRecord; - private MetricsRegistry registry = new MetricsRegistry(); - - /** Rate of operations applied by the sink */ - public final MetricsRate appliedOpsRate = - new MetricsRate("appliedOpsRate", registry); - - /** Rate of batches (of operations) applied by the sink */ - public final MetricsRate appliedBatchesRate = - new MetricsRate("appliedBatchesRate", registry); - - /** Age of the last operation that was applied by the sink */ - private final MetricsLongValue ageOfLastAppliedOp = - new MetricsLongValue("ageOfLastAppliedOp", registry); - - /** - * Constructor used to register the metrics - */ - public ReplicationSinkMetrics() { - MetricsContext context = MetricsUtil.getContext("hbase"); - String name = Thread.currentThread().getName(); - metricsRecord = MetricsUtil.createRecord(context, "replication"); - metricsRecord.setTag("RegionServer", name); - context.registerUpdater(this); - // export for JMX - new ReplicationStatistics(this.registry, "ReplicationSink"); - } - - /** - * Set the age of the last edit that was applied - * @param timestamp write time of the edit - */ - public void setAgeOfLastAppliedOp(long timestamp) { - ageOfLastAppliedOp.set(System.currentTimeMillis() - timestamp); - } - @Override - public void doUpdates(MetricsContext metricsContext) { - synchronized (this) { - this.appliedOpsRate.pushMetric(this.metricsRecord); - this.appliedBatchesRate.pushMetric(this.metricsRecord); - this.ageOfLastAppliedOp.pushMetric(this.metricsRecord); - } - this.metricsRecord.update(); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java index 2c328ea7258..ddca9d1bd69 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.replication.ReplicationZookeeper; +import org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationSourceMetrics; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; @@ -238,7 +239,7 @@ public class ReplicationSource extends Thread @Override public void enqueueLog(Path log) { this.queue.put(log); - this.metrics.sizeOfLogQueue.set(queue.size()); + this.metrics.setSizeOfLogQueue(queue.size()); } @Override @@ -246,6 +247,7 @@ public class ReplicationSource extends Thread connectToPeers(); // We were stopped while looping to connect to sinks, just abort if (!this.isActive()) { + metrics.clear(); return; } // delay this until we are in an asynchronous thread @@ -376,6 +378,7 @@ public class ReplicationSource extends Thread } } LOG.debug("Source exiting " + peerId); + metrics.clear(); } /** @@ -393,7 +396,7 @@ public class ReplicationSource extends Thread HLog.Entry entry = this.reader.next(this.entriesArray[currentNbEntries]); while (entry != null) { WALEdit edit = entry.getEdit(); - this.metrics.logEditsReadRate.inc(1); + this.metrics.incrLogEditsRead(); seenEntries++; // Remove all KVs that should not be replicated HLogKey logKey = entry.getKey(); @@ -415,7 +418,7 @@ public class ReplicationSource extends Thread currentNbOperations += countDistinctRowKeys(edit); currentNbEntries++; } else { - this.metrics.logEditsFilteredRate.inc(1); + this.metrics.incrLogEditsFiltered(); } } // Stop if too many entries or too big @@ -455,7 +458,7 @@ public class ReplicationSource extends Thread try { if (this.currentPath == null) { this.currentPath = queue.poll(this.sleepForRetries, TimeUnit.MILLISECONDS); - this.metrics.sizeOfLogQueue.set(queue.size()); + this.metrics.setSizeOfLogQueue(queue.size()); } } catch (InterruptedException e) { LOG.warn("Interrupted while reading edits", e); @@ -616,9 +619,7 @@ public class ReplicationSource extends Thread this.lastLoggedPosition = this.position; } this.totalReplicatedEdits += currentNbEntries; - this.metrics.shippedBatchesRate.inc(1); - this.metrics.shippedOpsRate.inc( - this.currentNbOperations); + this.metrics.shipBatch(this.currentNbOperations); this.metrics.setAgeOfLastShippedOp( this.entriesArray[this.entriesArray.length-1].getKey().getWriteTime()); LOG.debug("Replicated in total: " + this.totalReplicatedEdits); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java deleted file mode 100644 index 543e15df423..00000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceMetrics.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright 2010 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.replication.regionserver; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.metrics.MetricsRate; -import org.apache.hadoop.metrics.MetricsContext; -import org.apache.hadoop.metrics.MetricsRecord; -import org.apache.hadoop.metrics.MetricsUtil; -import org.apache.hadoop.metrics.Updater; -import org.apache.hadoop.metrics.jvm.JvmMetrics; -import org.apache.hadoop.metrics.util.MetricsIntValue; -import org.apache.hadoop.metrics.util.MetricsLongValue; -import org.apache.hadoop.metrics.util.MetricsRegistry; - -/** - * This class is for maintaining the various replication statistics - * for a source and publishing them through the metrics interfaces. - */ -@InterfaceAudience.Private -public class ReplicationSourceMetrics implements Updater { - private final MetricsRecord metricsRecord; - private MetricsRegistry registry = new MetricsRegistry(); - - /** Rate of shipped operations by the source */ - public final MetricsRate shippedOpsRate = - new MetricsRate("shippedOpsRate", registry); - - /** Rate of shipped batches by the source */ - public final MetricsRate shippedBatchesRate = - new MetricsRate("shippedBatchesRate", registry); - - /** Rate of log entries (can be multiple Puts) read from the logs */ - public final MetricsRate logEditsReadRate = - new MetricsRate("logEditsReadRate", registry); - - /** Rate of log entries filtered by the source */ - public final MetricsRate logEditsFilteredRate = - new MetricsRate("logEditsFilteredRate", registry); - - /** Age of the last operation that was shipped by the source */ - private final MetricsLongValue ageOfLastShippedOp = - new MetricsLongValue("ageOfLastShippedOp", registry); - - /** - * Current size of the queue of logs to replicate, - * excluding the one being processed at the moment - */ - public final MetricsIntValue sizeOfLogQueue = - new MetricsIntValue("sizeOfLogQueue", registry); - - // It's a little dirty to preset the age to now since if we fail - // to replicate the very first time then it will show that age instead - // of nothing (although that might not be good either). - private long lastTimestampForAge = System.currentTimeMillis(); - - /** - * Constructor used to register the metrics - * @param id Name of the source this class is monitoring - */ - public ReplicationSourceMetrics(String id) { - MetricsContext context = MetricsUtil.getContext("hbase"); - String name = Thread.currentThread().getName(); - metricsRecord = MetricsUtil.createRecord(context, "replication"); - metricsRecord.setTag("RegionServer", name); - context.registerUpdater(this); - try { - id = URLEncoder.encode(id, "UTF8"); - } catch (UnsupportedEncodingException e) { - id = "CAN'T ENCODE UTF8"; - } - // export for JMX - new ReplicationStatistics(this.registry, "ReplicationSource for " + id); - } - - /** - * Set the age of the last edit that was shipped - * @param timestamp write time of the edit - */ - public void setAgeOfLastShippedOp(long timestamp) { - lastTimestampForAge = timestamp; - ageOfLastShippedOp.set(System.currentTimeMillis() - lastTimestampForAge); - } - - /** - * Convenience method to use the last given timestamp to refresh the age - * of the last edit. Used when replication fails and need to keep that - * metric accurate. - */ - public void refreshAgeOfLastShippedOp() { - setAgeOfLastShippedOp(lastTimestampForAge); - } - - @Override - public void doUpdates(MetricsContext metricsContext) { - synchronized (this) { - this.shippedOpsRate.pushMetric(this.metricsRecord); - this.shippedBatchesRate.pushMetric(this.metricsRecord); - this.logEditsReadRate.pushMetric(this.metricsRecord); - this.logEditsFilteredRate.pushMetric(this.metricsRecord); - this.ageOfLastShippedOp.pushMetric(this.metricsRecord); - this.sizeOfLogQueue.pushMetric(this.metricsRecord); - } - this.metricsRecord.update(); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java deleted file mode 100644 index ceeff5f4897..00000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationStatistics.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2010 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.replication.regionserver; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.metrics.MetricsMBeanBase; -import org.apache.hadoop.metrics.util.MBeanUtil; -import org.apache.hadoop.metrics.util.MetricsRegistry; - -import javax.management.ObjectName; - -/** - * Exports metrics recorded by {@link ReplicationSourceMetrics} as an MBean - * for JMX monitoring. - */ -@InterfaceAudience.Private -public class ReplicationStatistics extends MetricsMBeanBase { - - private final ObjectName mbeanName; - - /** - * Constructor to register the MBean - * @param registry which rehistry to use - * @param name name to get to this bean - */ - public ReplicationStatistics(MetricsRegistry registry, String name) { - super(registry, name); - mbeanName = MBeanUtil.registerMBean("Replication", name, this); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSinkMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSinkMetrics.java new file mode 100644 index 00000000000..7bab049427c --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSinkMetrics.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication.regionserver.metrics; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSource; +import org.apache.hadoop.hbase.replication.regionserver.metrics.ReplicationMetricsSourceFactory; + +/** + * This class is for maintaining the various replication statistics for a sink and publishing them + * through the metrics interfaces. + */ +@InterfaceAudience.Private +public class ReplicationSinkMetrics { + + public static final String SINK_AGE_OF_LAST_APPLIED_OP = "sink.ageOfLastAppliedOp"; + public static final String SINK_APPLIED_BATCHES = "sink.appliedBatches"; + public static final String SINK_APPLIED_OPS = "sink.appliedOps"; + + private ReplicationMetricsSource rms; + + public ReplicationSinkMetrics() { + rms = ReplicationMetricsSourceFactory.getInstance(); + } + + /** + * Set the age of the last applied operation + * + * @param timestamp The timestamp of the last operation applied. + */ + public void setAgeOfLastAppliedOp(long timestamp) { + long age = System.currentTimeMillis() - timestamp; + rms.setGauge(SINK_AGE_OF_LAST_APPLIED_OP, age); + } + + /** + * Convience method to change metrics when a batch of operations are applied. + * + * @param batchSize + */ + public void applyBatch(long batchSize) { + rms.incCounters(SINK_APPLIED_BATCHES, 1); + rms.incCounters(SINK_APPLIED_OPS, batchSize); + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSourceMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSourceMetrics.java new file mode 100644 index 00000000000..35e3f55079c --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSourceMetrics.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication.regionserver.metrics; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * This class is for maintaining the various replication statistics for a source and publishing them + * through the metrics interfaces. + */ +@InterfaceAudience.Private +public class ReplicationSourceMetrics { + + public static final String SOURCE_SIZE_OF_LOG_QUEUE = "source.sizeOfLogQueue"; + public static final String SOURCE_AGE_OF_LAST_SHIPPED_OP = "source.ageOfLastShippedOp"; + public static final String SOURCE_LOG_EDITS_READ = "source.logEditsRead"; + public static final String SOURCE_LOG_EDITS_FILTERED = "source.logEditsFiltered"; + public static final String SOURCE_SHIPPED_BATCHES = "source.shippedBatches"; + public static final String SOURCE_SHIPPED_OPS = "source.shippedOps"; + + public static final Log LOG = LogFactory.getLog(ReplicationSourceMetrics.class); + private String id; + + private long lastTimestamp = 0; + private int lastQueueSize = 0; + + private String sizeOfLogQueKey; + private String ageOfLastShippedOpKey; + private String logEditsReadKey; + private String logEditsFilteredKey; + private final String shippedBatchesKey; + private final String shippedOpsKey; + + private ReplicationMetricsSource rms; + + /** + * Constructor used to register the metrics + * + * @param id Name of the source this class is monitoring + */ + public ReplicationSourceMetrics(String id) { + this.id = id; + + sizeOfLogQueKey = "source." + id + ".sizeOfLogQueue"; + ageOfLastShippedOpKey = "source." + id + ".ageOfLastShippedOp"; + logEditsReadKey = "source." + id + ".logEditsRead"; + logEditsFilteredKey = "source." + id + ".logEditsFiltered"; + shippedBatchesKey = "source." + this.id + ".shippedBatches"; + shippedOpsKey = "source." + this.id + ".shippedOps"; + rms = ReplicationMetricsSourceFactory.getInstance(); + } + + /** + * Set the age of the last edit that was shipped + * + * @param timestamp write time of the edit + */ + public void setAgeOfLastShippedOp(long timestamp) { + long age = System.currentTimeMillis() - timestamp; + rms.setGauge(ageOfLastShippedOpKey, age); + rms.setGauge(SOURCE_AGE_OF_LAST_SHIPPED_OP, age); + this.lastTimestamp = timestamp; + } + + /** + * Convenience method to use the last given timestamp to refresh the age of the last edit. Used + * when replication fails and need to keep that metric accurate. + */ + public void refreshAgeOfLastShippedOp() { + if (this.lastTimestamp > 0) { + setAgeOfLastShippedOp(this.lastTimestamp); + } + } + + /** + * Set the size of the log queue + * + * @param size the size. + */ + public void setSizeOfLogQueue(int size) { + rms.setGauge(sizeOfLogQueKey, size); + rms.incGauge(SOURCE_SIZE_OF_LOG_QUEUE, size - lastQueueSize); + lastQueueSize = size; + } + + /** + * Add on the the number of log edits read + * + * @param delta the number of log edits read. + */ + private void incrLogEditsRead(long delta) { + rms.incCounters(logEditsReadKey, delta); + rms.incCounters(SOURCE_LOG_EDITS_READ, delta); + } + + /** Increment the number of log edits read by one. */ + public void incrLogEditsRead() { + incrLogEditsRead(1); + } + + /** + * Add on the number of log edits filtered + * + * @param delta the number filtered. + */ + private void incrLogEditsFiltered(long delta) { + rms.incCounters(logEditsFilteredKey, delta); + rms.incCounters(SOURCE_LOG_EDITS_FILTERED, delta); + } + + /** The number of log edits filtered out. */ + public void incrLogEditsFiltered() { + incrLogEditsFiltered(1); + } + + /** + * Convience method to apply changes to metrics do to shipping a batch of logs. + * + * @param batchSize the size of the batch that was shipped to sinks. + */ + public void shipBatch(long batchSize) { + rms.incCounters(shippedBatchesKey, 1); + rms.incCounters(SOURCE_SHIPPED_BATCHES, 1); + rms.incCounters(shippedOpsKey, batchSize); + rms.incCounters(SOURCE_SHIPPED_OPS, batchSize); + } + + /** Removes all metrics about this Source. */ + public void clear() { + rms.removeGauge(sizeOfLogQueKey); + rms.decGauge(SOURCE_SIZE_OF_LOG_QUEUE, lastQueueSize); + lastQueueSize = 0; + rms.removeGauge(ageOfLastShippedOpKey); + + rms.removeCounter(logEditsFilteredKey); + rms.removeCounter(logEditsReadKey); + + } +} diff --git a/pom.xml b/pom.xml index 9711fd800bc..fb05924b41f 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,9 @@ http://hbase.apache.org hbase-server + hbase-hadoop2-compat + hbase-hadoop1-compat + hbase-hadoop-compat hbase-common hbase-it @@ -600,7 +603,7 @@ gnu false - src/assembly/all.xml + ${assembly.file} @@ -786,6 +789,8 @@ 1.6 ${project.version} + 2.0.0-alpha + 1.0.3 1.5.3 1.2 1.4 @@ -837,6 +842,7 @@ hbase-server-${project.version}-tests.jar 2.12-TRUNK-HBASE-2 surefire-junit47 + hbase-hadoop1-compat false false @@ -865,6 +871,16 @@ hbase-common ${project.version} + + org.apache.hbase + hbase-hadoop-compat + ${project.version} + + + org.apache.hbase + ${compat.module} + ${project.version} + hbase-server org.apache.hbase @@ -1227,8 +1243,10 @@ - 1.0.3 + ${hadoop-one.version} 1.4.3 + hbase-hadoop1-compat + src/assembly/hadoop-one-compat.xml @@ -1282,26 +1300,28 @@ 1.6.1 - 2.0.0-alpha + ${hadoop-two.version} + hbase-hadoop2-compat + src/assembly/hadoop-two-compat.xml org.apache.hadoop hadoop-client - ${hadoop.version} + ${hadoop-two.version} org.apache.hadoop hadoop-annotations - ${hadoop.version} + ${hadoop-two.version} org.apache.hadoop hadoop-minicluster - ${hadoop.version} + ${hadoop-two.version} diff --git a/src/assembly/all.xml b/src/assembly/hadoop-one-compat.xml similarity index 95% rename from src/assembly/all.xml rename to src/assembly/hadoop-one-compat.xml index 5a9866b6458..c3b661000a5 100644 --- a/src/assembly/all.xml +++ b/src/assembly/hadoop-one-compat.xml @@ -124,7 +124,10 @@ - org.apache.hbase:hbase-* + org.apache.hbase:hbase-common + org.apache.hbase:hbase-hadoop-compat + org.apache.hbase:hbase-hadoop1-compat + org.apache.hbase:hbase-server diff --git a/src/assembly/hadoop-two-compat.xml b/src/assembly/hadoop-two-compat.xml new file mode 100644 index 00000000000..9f5a8443beb --- /dev/null +++ b/src/assembly/hadoop-two-compat.xml @@ -0,0 +1,156 @@ + + + + + + all + + tar.gz + + + + + target/site + docs + + + + / + + *.txt + pom.xml + + + + + conf + conf + 0644 + 0755 + + + + bin + bin + 0755 + 0755 + + + conf + conf + 0644 + 0755 + + + + src + src + 0644 + 0755 + + + + dev-support + dev-support + 0755 + 0755 + + + + hbase-server/src/main/ruby + lib/ruby + 0644 + 0755 + + + + hbase-server/target/hbase-webapps + hbase-webapps + 0644 + 0755 + + + + hbase-server/target/native + native + 0755 + 0755 + + *.so + + + + + ${parent.basedir}/hbase-server/target/ + lib + + ${server.test.jar} + + 0644 + + + + + + true + + + + + org.apache.hbase:hbase-common + org.apache.hbase:hbase-hadoop-compat + org.apache.hbase:hbase-hadoop2-compat + org.apache.hbase:hbase-server + + + + + + + target/ + test/ + .classpath + .project + .settings/ + + + + + + + lib + false + + + + + + +