From f955d52f39c1ecb4aa53cc4f4d18652b4914d8ee Mon Sep 17 00:00:00 2001 From: Eric E Payne Date: Thu, 12 Apr 2018 16:04:23 +0000 Subject: [PATCH] YARN-8120. JVM can crash with SIGSEGV when exiting due to custom leveldb logger. Contributed by Jason Lowe. (cherry picked from commit 6bb128dfb893cf0e4aa2d3ecc65440668a1fc8d7) --- .../v2/hs/HistoryServerLeveldbStateStoreService.java | 11 ----------- .../org/apache/hadoop/mapred/ShuffleHandler.java | 12 ------------ .../recovery/NMLeveldbStateStoreService.java | 12 ------------ .../recovery/LeveldbRMStateStore.java | 12 ------------ 4 files changed, 47 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java index c8741aaf623..6d2e4075950 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java @@ -74,7 +74,6 @@ public class HistoryServerLeveldbStateStoreService extends Path storeRoot = createStorageDir(getConfig()); Options options = new Options(); options.createIfMissing(false); - options.logger(new LeveldbLogger()); LOG.info("Using state database at " + storeRoot + " for recovery"); File dbfile = new File(storeRoot.toString()); try { @@ -366,14 +365,4 @@ public class HistoryServerLeveldbStateStoreService extends + getCurrentVersion() + ", but loading version " + loadedVersion); } } - - private static class LeveldbLogger implements org.iq80.leveldb.Logger { - private static final Logger LOG = - LoggerFactory.getLogger(LeveldbLogger.class); - - @Override - public void log(String message) { - LOG.info(message); - } - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index ec992feeb77..aeda9cc688d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -93,7 +93,6 @@ import org.fusesource.leveldbjni.JniDBFactory; import org.fusesource.leveldbjni.internal.NativeDB; import org.iq80.leveldb.DB; import org.iq80.leveldb.DBException; -import org.iq80.leveldb.Logger; import org.iq80.leveldb.Options; import org.jboss.netty.bootstrap.ServerBootstrap; import org.jboss.netty.buffer.ChannelBuffers; @@ -628,7 +627,6 @@ public class ShuffleHandler extends AuxiliaryService { private void startStore(Path recoveryRoot) throws IOException { Options options = new Options(); options.createIfMissing(false); - options.logger(new LevelDBLogger()); Path dbPath = new Path(recoveryRoot, STATE_DB_NAME); LOG.info("Using state database at " + dbPath + " for recovery"); File dbfile = new File(dbPath.toString()); @@ -774,16 +772,6 @@ public class ShuffleHandler extends AuxiliaryService { } } - private static class LevelDBLogger implements Logger { - private static final org.slf4j.Logger LOG = - LoggerFactory.getLogger(LevelDBLogger.class); - - @Override - public void log(String message) { - LOG.info(message); - } - } - static class TimeoutHandler extends IdleStateAwareChannelHandler { private boolean enabledTimeout; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java index 723dd48471e..6f643b04d5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java @@ -1531,7 +1531,6 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { Path storeRoot = createStorageDir(conf); Options options = new Options(); options.createIfMissing(false); - options.logger(new LeveldbLogger()); LOG.info("Using state database at " + storeRoot + " for recovery"); File dbfile = new File(storeRoot.toString()); try { @@ -1596,17 +1595,6 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { } } - private static class LeveldbLogger implements org.iq80.leveldb.Logger { - private static final org.slf4j.Logger LOG = - LoggerFactory.getLogger(LeveldbLogger.class); - - @Override - public void log(String message) { - LOG.info(message); - } - } - - Version loadVersion() throws IOException { byte[] data = db.get(bytes(DB_SCHEMA_VERSION_KEY)); // if version is not stored previously, treat it as CURRENT_VERSION_INFO. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java index a53083f9b22..36a8dfa8b8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java @@ -64,12 +64,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AM import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; import org.apache.hadoop.yarn.server.utils.LeveldbIterator; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.fusesource.leveldbjni.JniDBFactory; import org.fusesource.leveldbjni.internal.NativeDB; import org.iq80.leveldb.DB; import org.iq80.leveldb.DBException; -import org.iq80.leveldb.Logger; import org.iq80.leveldb.Options; import org.iq80.leveldb.WriteBatch; @@ -165,7 +163,6 @@ public class LeveldbRMStateStore extends RMStateStore { Path storeRoot = createStorageDir(); Options options = new Options(); options.createIfMissing(false); - options.logger(new LeveldbLogger()); LOG.info("Using state database at " + storeRoot + " for recovery"); File dbfile = new File(storeRoot.toString()); try { @@ -872,13 +869,4 @@ public class LeveldbRMStateStore extends RMStateStore { LOG.info("Full compaction cycle completed in " + duration + " msec"); } } - - private static class LeveldbLogger implements Logger { - private static final Log LOG = LogFactory.getLog(LeveldbLogger.class); - - @Override - public void log(String message) { - LOG.info(message); - } - } }