HADOOP-1813 [hbase] OOME makes zombie of region server
Forgot to add below as part of last commit. git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@575950 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack
parent
83298b4721
commit
7ce0506c27
|
|
@ -0,0 +1,85 @@
|
|||
/**
|
||||
* Copyright 2007 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.hbase.util.Sleeper;
|
||||
import org.apache.hadoop.hbase.util.Threads;
|
||||
|
||||
/**
|
||||
* Chore is a task performed on a period in hbase. The chore is run in its own
|
||||
* thread. This base abstract class provides while loop and sleeping facility.
|
||||
* If an unhandled exception, the threads exit is logged.
|
||||
* Implementers just need to add checking if there is work to be done and if
|
||||
* so, do it. Its the base of most of the chore threads in hbase.
|
||||
*/
|
||||
public abstract class Chore extends Thread {
|
||||
private final Log LOG = LogFactory.getLog(this.getClass());
|
||||
private final Sleeper sleeper;
|
||||
protected final AtomicBoolean stop;
|
||||
|
||||
/**
|
||||
* @param p Period at which we should run. Will be adjusted appropriately
|
||||
* should we find work and it takes time to complete.
|
||||
* @param s When this flag is set to true, this thread will cleanup and exit
|
||||
* cleanly.
|
||||
*/
|
||||
public Chore(final int p, final AtomicBoolean s) {
|
||||
super();
|
||||
this.sleeper = new Sleeper(p, s);
|
||||
this.stop = s;
|
||||
}
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
initialChore();
|
||||
this.sleeper.sleep();
|
||||
while(!this.stop.get()) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
chore();
|
||||
this.sleeper.sleep(startTime);
|
||||
}
|
||||
} finally {
|
||||
LOG.info(getName() + " exiting");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Override to run a task before we start looping.
|
||||
*/
|
||||
protected void initialChore() {
|
||||
// Default does nothing.
|
||||
}
|
||||
|
||||
/**
|
||||
* Look for chores. If any found, do them else just return.
|
||||
*/
|
||||
protected abstract void chore();
|
||||
|
||||
/**
|
||||
* Sleep for period.
|
||||
*/
|
||||
protected void sleep() {
|
||||
this.sleeper.sleep();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/**
|
||||
* Copyright 2007 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
/**
|
||||
* Sleeper for current thread.
|
||||
* Sleeps for passed period. Also checks passed boolean and if interrupted,
|
||||
* will return if the flag is set (rather than go back to sleep until its
|
||||
* sleep time is up).
|
||||
*/
|
||||
public class Sleeper {
|
||||
private final int period;
|
||||
private AtomicBoolean stop;
|
||||
|
||||
public Sleeper(final int sleep, final AtomicBoolean stop) {
|
||||
this.period = sleep;
|
||||
this.stop = stop;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep for period.
|
||||
*/
|
||||
public void sleep() {
|
||||
sleep(System.currentTimeMillis());
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep for period adjusted by passed <code>startTime<code>
|
||||
* @param startTime Time some task started previous to now. Time to sleep
|
||||
* will be docked current time minus passed <code>startTime<code>.
|
||||
*/
|
||||
public void sleep(final long startTime) {
|
||||
if (this.stop.get()) {
|
||||
return;
|
||||
}
|
||||
long waitTime = this.period - (System.currentTimeMillis() - startTime);
|
||||
if (waitTime > 0) {
|
||||
try {
|
||||
Thread.sleep(waitTime);
|
||||
} catch(InterruptedException iex) {
|
||||
// We we interrupted because we're meant to stop? If not, just
|
||||
// continue ignoring the interruption
|
||||
if (this.stop.get()) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.lang.Thread.UncaughtExceptionHandler;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
/**
|
||||
* Thread Utility
|
||||
*/
|
||||
public class Threads {
|
||||
protected static final Log LOG = LogFactory.getLog(Threads.class);
|
||||
|
||||
/**
|
||||
* Utility method that sets name, daemon status and starts passed thread.
|
||||
* @param t
|
||||
* @param name
|
||||
* @return Returns the passed Thread <code>t</code>.
|
||||
*/
|
||||
public static Thread setDaemonThreadRunning(final Thread t,
|
||||
final String name) {
|
||||
return setDaemonThreadRunning(t, name, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility method that sets name, daemon status and starts passed thread.
|
||||
* @param t
|
||||
* @param name
|
||||
* @param handler A handler to set on the thread. Pass null if want to
|
||||
* use default handler.
|
||||
* @return Returns the passed Thread <code>t</code>.
|
||||
*/
|
||||
public static Thread setDaemonThreadRunning(final Thread t,
|
||||
final String name, final UncaughtExceptionHandler handler) {
|
||||
t.setName(name);
|
||||
if (handler != null) {
|
||||
t.setUncaughtExceptionHandler(handler);
|
||||
}
|
||||
t.setDaemon(true);
|
||||
t.start();
|
||||
return t;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
/**
|
||||
* Copyright 2007 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
||||
/**
|
||||
* An HMaster that runs out of memory.
|
||||
* Everytime a region server reports in, add to the retained heap of memory.
|
||||
* Needs to be started manually as in
|
||||
* <code>${HBASE_HOME}/bin/hbase ./bin/hbase org.apache.hadoop.hbase.OOMEHMaster start/code>.
|
||||
*/
|
||||
public class OOMEHMaster extends HMaster {
|
||||
private List<byte []> retainer = new ArrayList<byte[]>();
|
||||
|
||||
public OOMEHMaster(Configuration conf) throws IOException {
|
||||
super(conf);
|
||||
}
|
||||
|
||||
public OOMEHMaster(Path dir, HServerAddress address, Configuration conf)
|
||||
throws IOException {
|
||||
super(dir, address, conf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HMsg[] regionServerReport(HServerInfo serverInfo, HMsg[] msgs)
|
||||
throws IOException {
|
||||
// Retain 1M.
|
||||
this.retainer.add(new byte [1024 * 1024]);
|
||||
return super.regionServerReport(serverInfo, msgs);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param args
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
doMain(args, OOMEHMaster.class);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
/**
|
||||
* Copyright 2007 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.io.BatchUpdate;
|
||||
import org.apache.hadoop.io.Text;
|
||||
|
||||
/**
|
||||
* A region server that will OOME.
|
||||
* Everytime {@link #batchUpdate(Text, long, BatchUpdate)} is called, we add
|
||||
* keep around a reference to the batch. Use this class to test OOME extremes.
|
||||
* Needs to be started manually as in
|
||||
* <code>${HBASE_HOME}/bin/hbase ./bin/hbase org.apache.hadoop.hbase.OOMERegionServer start</code>.
|
||||
*/
|
||||
public class OOMERegionServer extends HRegionServer {
|
||||
private List<BatchUpdate> retainer = new ArrayList<BatchUpdate>();
|
||||
|
||||
public OOMERegionServer(Configuration conf) throws IOException {
|
||||
super(conf);
|
||||
}
|
||||
|
||||
public OOMERegionServer(Path rootDir, HServerAddress address,
|
||||
Configuration conf) throws IOException {
|
||||
super(rootDir, address, conf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void batchUpdate(Text regionName, long timestamp, BatchUpdate b)
|
||||
throws IOException {
|
||||
super.batchUpdate(regionName, timestamp, b);
|
||||
for (int i = 0; i < 30; i++) {
|
||||
// Add the batch update 30 times to bring on the OOME faster.
|
||||
this.retainer.add(b);
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
HRegionServer.doMain(args, OOMERegionServer.class);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue