HADOOP-1813 [hbase] OOME makes zombie of region server
Forgot to add below as part of last commit. git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@575950 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack
parent
83298b4721
commit
7ce0506c27
|
|
@ -0,0 +1,85 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2007 The Apache Software Foundation
|
||||||
|
*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase;
|
||||||
|
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.hbase.util.Sleeper;
|
||||||
|
import org.apache.hadoop.hbase.util.Threads;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Chore is a task performed on a period in hbase. The chore is run in its own
|
||||||
|
* thread. This base abstract class provides while loop and sleeping facility.
|
||||||
|
* If an unhandled exception, the threads exit is logged.
|
||||||
|
* Implementers just need to add checking if there is work to be done and if
|
||||||
|
* so, do it. Its the base of most of the chore threads in hbase.
|
||||||
|
*/
|
||||||
|
public abstract class Chore extends Thread {
|
||||||
|
private final Log LOG = LogFactory.getLog(this.getClass());
|
||||||
|
private final Sleeper sleeper;
|
||||||
|
protected final AtomicBoolean stop;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param p Period at which we should run. Will be adjusted appropriately
|
||||||
|
* should we find work and it takes time to complete.
|
||||||
|
* @param s When this flag is set to true, this thread will cleanup and exit
|
||||||
|
* cleanly.
|
||||||
|
*/
|
||||||
|
public Chore(final int p, final AtomicBoolean s) {
|
||||||
|
super();
|
||||||
|
this.sleeper = new Sleeper(p, s);
|
||||||
|
this.stop = s;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
try {
|
||||||
|
initialChore();
|
||||||
|
this.sleeper.sleep();
|
||||||
|
while(!this.stop.get()) {
|
||||||
|
long startTime = System.currentTimeMillis();
|
||||||
|
chore();
|
||||||
|
this.sleeper.sleep(startTime);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
LOG.info(getName() + " exiting");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Override to run a task before we start looping.
|
||||||
|
*/
|
||||||
|
protected void initialChore() {
|
||||||
|
// Default does nothing.
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Look for chores. If any found, do them else just return.
|
||||||
|
*/
|
||||||
|
protected abstract void chore();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sleep for period.
|
||||||
|
*/
|
||||||
|
protected void sleep() {
|
||||||
|
this.sleeper.sleep();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,68 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2007 The Apache Software Foundation
|
||||||
|
*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.util;
|
||||||
|
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sleeper for current thread.
|
||||||
|
* Sleeps for passed period. Also checks passed boolean and if interrupted,
|
||||||
|
* will return if the flag is set (rather than go back to sleep until its
|
||||||
|
* sleep time is up).
|
||||||
|
*/
|
||||||
|
public class Sleeper {
|
||||||
|
private final int period;
|
||||||
|
private AtomicBoolean stop;
|
||||||
|
|
||||||
|
public Sleeper(final int sleep, final AtomicBoolean stop) {
|
||||||
|
this.period = sleep;
|
||||||
|
this.stop = stop;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sleep for period.
|
||||||
|
*/
|
||||||
|
public void sleep() {
|
||||||
|
sleep(System.currentTimeMillis());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sleep for period adjusted by passed <code>startTime<code>
|
||||||
|
* @param startTime Time some task started previous to now. Time to sleep
|
||||||
|
* will be docked current time minus passed <code>startTime<code>.
|
||||||
|
*/
|
||||||
|
public void sleep(final long startTime) {
|
||||||
|
if (this.stop.get()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
long waitTime = this.period - (System.currentTimeMillis() - startTime);
|
||||||
|
if (waitTime > 0) {
|
||||||
|
try {
|
||||||
|
Thread.sleep(waitTime);
|
||||||
|
} catch(InterruptedException iex) {
|
||||||
|
// We we interrupted because we're meant to stop? If not, just
|
||||||
|
// continue ignoring the interruption
|
||||||
|
if (this.stop.get()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,60 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.util;
|
||||||
|
|
||||||
|
import java.lang.Thread.UncaughtExceptionHandler;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Thread Utility
|
||||||
|
*/
|
||||||
|
public class Threads {
|
||||||
|
protected static final Log LOG = LogFactory.getLog(Threads.class);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Utility method that sets name, daemon status and starts passed thread.
|
||||||
|
* @param t
|
||||||
|
* @param name
|
||||||
|
* @return Returns the passed Thread <code>t</code>.
|
||||||
|
*/
|
||||||
|
public static Thread setDaemonThreadRunning(final Thread t,
|
||||||
|
final String name) {
|
||||||
|
return setDaemonThreadRunning(t, name, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Utility method that sets name, daemon status and starts passed thread.
|
||||||
|
* @param t
|
||||||
|
* @param name
|
||||||
|
* @param handler A handler to set on the thread. Pass null if want to
|
||||||
|
* use default handler.
|
||||||
|
* @return Returns the passed Thread <code>t</code>.
|
||||||
|
*/
|
||||||
|
public static Thread setDaemonThreadRunning(final Thread t,
|
||||||
|
final String name, final UncaughtExceptionHandler handler) {
|
||||||
|
t.setName(name);
|
||||||
|
if (handler != null) {
|
||||||
|
t.setUncaughtExceptionHandler(handler);
|
||||||
|
}
|
||||||
|
t.setDaemon(true);
|
||||||
|
t.start();
|
||||||
|
return t;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,61 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2007 The Apache Software Foundation
|
||||||
|
*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An HMaster that runs out of memory.
|
||||||
|
* Everytime a region server reports in, add to the retained heap of memory.
|
||||||
|
* Needs to be started manually as in
|
||||||
|
* <code>${HBASE_HOME}/bin/hbase ./bin/hbase org.apache.hadoop.hbase.OOMEHMaster start/code>.
|
||||||
|
*/
|
||||||
|
public class OOMEHMaster extends HMaster {
|
||||||
|
private List<byte []> retainer = new ArrayList<byte[]>();
|
||||||
|
|
||||||
|
public OOMEHMaster(Configuration conf) throws IOException {
|
||||||
|
super(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
public OOMEHMaster(Path dir, HServerAddress address, Configuration conf)
|
||||||
|
throws IOException {
|
||||||
|
super(dir, address, conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public HMsg[] regionServerReport(HServerInfo serverInfo, HMsg[] msgs)
|
||||||
|
throws IOException {
|
||||||
|
// Retain 1M.
|
||||||
|
this.retainer.add(new byte [1024 * 1024]);
|
||||||
|
return super.regionServerReport(serverInfo, msgs);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param args
|
||||||
|
*/
|
||||||
|
public static void main(String[] args) {
|
||||||
|
doMain(args, OOMEHMaster.class);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,63 @@
|
||||||
|
/**
|
||||||
|
* Copyright 2007 The Apache Software Foundation
|
||||||
|
*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.hbase.io.BatchUpdate;
|
||||||
|
import org.apache.hadoop.io.Text;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A region server that will OOME.
|
||||||
|
* Everytime {@link #batchUpdate(Text, long, BatchUpdate)} is called, we add
|
||||||
|
* keep around a reference to the batch. Use this class to test OOME extremes.
|
||||||
|
* Needs to be started manually as in
|
||||||
|
* <code>${HBASE_HOME}/bin/hbase ./bin/hbase org.apache.hadoop.hbase.OOMERegionServer start</code>.
|
||||||
|
*/
|
||||||
|
public class OOMERegionServer extends HRegionServer {
|
||||||
|
private List<BatchUpdate> retainer = new ArrayList<BatchUpdate>();
|
||||||
|
|
||||||
|
public OOMERegionServer(Configuration conf) throws IOException {
|
||||||
|
super(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
public OOMERegionServer(Path rootDir, HServerAddress address,
|
||||||
|
Configuration conf) throws IOException {
|
||||||
|
super(rootDir, address, conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void batchUpdate(Text regionName, long timestamp, BatchUpdate b)
|
||||||
|
throws IOException {
|
||||||
|
super.batchUpdate(regionName, timestamp, b);
|
||||||
|
for (int i = 0; i < 30; i++) {
|
||||||
|
// Add the batch update 30 times to bring on the OOME faster.
|
||||||
|
this.retainer.add(b);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
HRegionServer.doMain(args, OOMERegionServer.class);
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue