diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 066db20f89a..d358f529bc8 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -93,6 +93,9 @@ Release 2.5.0 - UNRELEASED YARN-2073. Fair Scheduler: Add a utilization threshold to prevent preempting resources when cluster is free (Karthik Kambatla via Sandy Ryza) + YARN-2071. Modified levelDB store permissions to be readable only by the + server user. (Zhijie Shen via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java index edd48426976..51bffda6274 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.readReverseOrderedLong; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; + import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -36,13 +39,16 @@ import java.util.TreeMap; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.google.common.annotations.VisibleForTesting; import org.apache.commons.collections.map.LRUMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.service.AbstractService; @@ -50,8 +56,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.fusesource.leveldbjni.JniDBFactory; @@ -62,8 +68,7 @@ import org.iq80.leveldb.ReadOptions; import org.iq80.leveldb.WriteBatch; import org.iq80.leveldb.WriteOptions; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.readReverseOrderedLong; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; +import com.google.common.annotations.VisibleForTesting; /** *

An implementation of an application timeline store backed by leveldb.

@@ -120,7 +125,9 @@ public class LeveldbTimelineStore extends AbstractService private static final Log LOG = LogFactory .getLog(LeveldbTimelineStore.class); - private static final String FILENAME = "leveldb-timeline-store.ldb"; + @Private + @VisibleForTesting + static final String FILENAME = "leveldb-timeline-store.ldb"; private static final byte[] START_TIME_LOOKUP_PREFIX = "k".getBytes(); private static final byte[] ENTITY_ENTRY_PREFIX = "e".getBytes(); @@ -135,6 +142,11 @@ public class LeveldbTimelineStore extends AbstractService private static final byte[] EMPTY_BYTES = new byte[0]; + @Private + @VisibleForTesting + static final FsPermission LEVELDB_DIR_UMASK = FsPermission + .createImmutable((short) 0700); + private Map startTimeWriteCache; private Map startTimeReadCache; @@ -164,16 +176,23 @@ public class LeveldbTimelineStore extends AbstractService YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE)); JniDBFactory factory = new JniDBFactory(); - String path = conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH); - File p = new File(path); - if (!p.exists()) { - if (!p.mkdirs()) { - throw new IOException("Couldn't create directory for leveldb " + - "timeline store " + path); + Path dbPath = new Path( + conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH), FILENAME); + FileSystem localFS = null; + try { + localFS = FileSystem.getLocal(conf); + if (!localFS.exists(dbPath)) { + if (!localFS.mkdirs(dbPath)) { + throw new IOException("Couldn't create directory for leveldb " + + "timeline store " + dbPath); + } + localFS.setPermission(dbPath, LEVELDB_DIR_UMASK); } + } finally { + IOUtils.cleanup(LOG, localFS); } - LOG.info("Using leveldb path " + path); - db = factory.open(new File(path, FILENAME), options); + LOG.info("Using leveldb path " + dbPath); + db = factory.open(new File(dbPath.toString()), options); startTimeWriteCache = Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize( conf))); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java index 9b273090bd0..59449f82dc1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import java.io.File; import java.io.IOException; import java.util.Collections; @@ -28,6 +32,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; @@ -39,9 +45,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; -import static org.junit.Assert.assertEquals; - @InterfaceAudience.Private @InterfaceStability.Unstable public class TestLeveldbTimelineStore extends TimelineStoreTestUtils { @@ -51,7 +54,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils { @Before public void setup() throws Exception { fsContext = FileContext.getLocalFSFileContext(); - Configuration conf = new Configuration(); + Configuration conf = new YarnConfiguration(); fsPath = new File("target", this.getClass().getSimpleName() + "-tmpDir").getAbsoluteFile(); fsContext.delete(new Path(fsPath.getAbsolutePath()), true); @@ -71,6 +74,15 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils { fsContext.delete(new Path(fsPath.getAbsolutePath()), true); } + @Test + public void testRootDirPermission() throws IOException { + FileSystem fs = FileSystem.getLocal(new YarnConfiguration()); + FileStatus file = fs.getFileStatus( + new Path(fsPath.getAbsolutePath(), LeveldbTimelineStore.FILENAME)); + assertNotNull(file); + assertEquals(LeveldbTimelineStore.LEVELDB_DIR_UMASK, file.getPermission()); + } + @Test public void testGetSingleEntity() throws IOException { super.testGetSingleEntity();