From 3671a5e16fbddbe5a0516289ce98e1305e02291c Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Thu, 22 May 2014 18:17:11 +0000 Subject: [PATCH 01/34] MAPREDUCE-5899. Support incremental data copy in DistCp. Contributed by Jing Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1596931 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/hadoop/fs/FileSystem.java | 14 +- .../apache/hadoop/fs/FilterFileSystem.java | 8 +- .../org/apache/hadoop/fs/HarFileSystem.java | 2 +- .../apache/hadoop/fs/TestHarFileSystem.java | 11 +- .../main/java/org/apache/hadoop/fs/Hdfs.java | 2 +- .../org/apache/hadoop/hdfs/DFSClient.java | 35 ++-- .../hadoop/hdfs/DistributedFileSystem.java | 31 ++- .../hdfs/server/datanode/DataXceiver.java | 77 +++++-- .../web/resources/DatanodeWebHdfsMethods.java | 3 +- .../hadoop/hdfs/TestGetFileChecksum.java | 75 +++++++ hadoop-mapreduce-project/CHANGES.txt | 2 + .../apache/hadoop/tools/DistCpConstants.java | 1 + .../hadoop/tools/DistCpOptionSwitch.java | 4 + .../apache/hadoop/tools/DistCpOptions.java | 28 +++ .../apache/hadoop/tools/OptionsParser.java | 4 + .../hadoop/tools/mapred/CopyMapper.java | 105 ++++++---- .../mapred/RetriableFileCopyCommand.java | 120 ++++++----- .../tools/util/ThrottledInputStream.java | 24 +++ .../hadoop/tools/TestOptionsParser.java | 44 ++++ .../hadoop/tools/mapred/TestCopyMapper.java | 195 ++++++++++++------ .../mapred/TestRetriableFileCopyCommand.java | 5 +- 21 files changed, 597 insertions(+), 193 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index d2f38cbdf47..b79196eed52 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2140,9 +2140,21 @@ public abstract class FileSystem extends Configured implements Closeable { * in the corresponding FileSystem. */ public FileChecksum getFileChecksum(Path f) throws IOException { + return getFileChecksum(f, Long.MAX_VALUE); + } + + /** + * Get the checksum of a file, from the beginning of the file till the + * specific length. + * @param f The file path + * @param length The length of the file range for checksum calculation + * @return The file checksum. + */ + public FileChecksum getFileChecksum(Path f, final long length) + throws IOException { return null; } - + /** * Set the verify checksum flag. This is only applicable if the * corresponding FileSystem supports checksum. By default doesn't do anything. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index ac094272dd9..b98ff40ab56 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -31,7 +31,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.Progressable; @@ -428,7 +427,12 @@ public class FilterFileSystem extends FileSystem { public FileChecksum getFileChecksum(Path f) throws IOException { return fs.getFileChecksum(f); } - + + @Override + public FileChecksum getFileChecksum(Path f, long length) throws IOException { + return fs.getFileChecksum(f, length); + } + @Override public void setVerifyChecksum(boolean verifyChecksum) { fs.setVerifyChecksum(verifyChecksum); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java index 04780d0911e..3ba6de1f9a3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java @@ -687,7 +687,7 @@ public class HarFileSystem extends FileSystem { * @return null since no checksum algorithm is implemented. */ @Override - public FileChecksum getFileChecksum(Path f) { + public FileChecksum getFileChecksum(Path f, long length) { return null; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java index 1614d6b1c6d..16db5b11160 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java @@ -139,6 +139,7 @@ public class TestHarFileSystem { public int getDefaultPort(); public String getCanonicalServiceName(); public Token getDelegationToken(String renewer) throws IOException; + public FileChecksum getFileChecksum(Path f) throws IOException; public boolean deleteOnExit(Path f) throws IOException; public boolean cancelDeleteOnExit(Path f) throws IOException; public Token[] addDelegationTokens(String renewer, Credentials creds) @@ -223,10 +224,16 @@ public class TestHarFileSystem { } @Test - public void testFileChecksum() { + public void testFileChecksum() throws Exception { final Path p = new Path("har://file-localhost/foo.har/file1"); final HarFileSystem harfs = new HarFileSystem(); - Assert.assertEquals(null, harfs.getFileChecksum(p)); + try { + Assert.assertEquals(null, harfs.getFileChecksum(p)); + } finally { + if (harfs != null) { + harfs.close(); + } + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java index 4f51fc1f7e3..e308a966f55 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -116,7 +116,7 @@ public class Hdfs extends AbstractFileSystem { @Override public FileChecksum getFileChecksum(Path f) throws IOException, UnresolvedLinkException { - return dfs.getFileChecksum(getUriPath(f)); + return dfs.getFileChecksum(getUriPath(f), Long.MAX_VALUE); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 38eca806d1c..5c7095931ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -1801,15 +1801,19 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory { } /** - * Get the checksum of a file. + * Get the checksum of the whole file of a range of the file. Note that the + * range always starts from the beginning of the file. * @param src The file path + * @param length The length of the range * @return The checksum * @see DistributedFileSystem#getFileChecksum(Path) */ - public MD5MD5CRC32FileChecksum getFileChecksum(String src) throws IOException { + public MD5MD5CRC32FileChecksum getFileChecksum(String src, long length) + throws IOException { checkOpen(); - return getFileChecksum(src, clientName, namenode, socketFactory, - dfsClientConf.socketTimeout, getDataEncryptionKey(), + Preconditions.checkArgument(length >= 0); + return getFileChecksum(src, length, clientName, namenode, + socketFactory, dfsClientConf.socketTimeout, getDataEncryptionKey(), dfsClientConf.connectToDnViaHostname); } @@ -1850,8 +1854,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory { } /** - * Get the checksum of a file. + * Get the checksum of the whole file or a range of the file. * @param src The file path + * @param length the length of the range, i.e., the range is [0, length] * @param clientName the name of the client requesting the checksum. * @param namenode the RPC proxy for the namenode * @param socketFactory to create sockets to connect to DNs @@ -1861,12 +1866,13 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory { * @return The checksum */ private static MD5MD5CRC32FileChecksum getFileChecksum(String src, - String clientName, - ClientProtocol namenode, SocketFactory socketFactory, int socketTimeout, + long length, String clientName, ClientProtocol namenode, + SocketFactory socketFactory, int socketTimeout, DataEncryptionKey encryptionKey, boolean connectToDnViaHostname) throws IOException { - //get all block locations - LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE); + //get block locations for the file range + LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0, + length); if (null == blockLocations) { throw new FileNotFoundException("File does not exist: " + src); } @@ -1878,10 +1884,11 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory { boolean refetchBlocks = false; int lastRetriedIndex = -1; - //get block checksum for each block - for(int i = 0; i < locatedblocks.size(); i++) { + // get block checksum for each block + long remaining = length; + for(int i = 0; i < locatedblocks.size() && remaining > 0; i++) { if (refetchBlocks) { // refetch to get fresh tokens - blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE); + blockLocations = callGetBlockLocations(namenode, src, 0, length); if (null == blockLocations) { throw new FileNotFoundException("File does not exist: " + src); } @@ -1890,6 +1897,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory { } LocatedBlock lb = locatedblocks.get(i); final ExtendedBlock block = lb.getBlock(); + if (remaining < block.getNumBytes()) { + block.setNumBytes(remaining); + } + remaining -= block.getNumBytes(); final DatanodeInfo[] datanodes = lb.getLocations(); //try each datanode location of the block diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 38bb95a163f..1f577271b44 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -68,14 +68,12 @@ import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; @@ -85,7 +83,6 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; -import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; @@ -1142,7 +1139,7 @@ public class DistributedFileSystem extends FileSystem { @Override public FileChecksum doCall(final Path p) throws IOException, UnresolvedLinkException { - return dfs.getFileChecksum(getPathName(p)); + return dfs.getFileChecksum(getPathName(p), Long.MAX_VALUE); } @Override @@ -1153,6 +1150,32 @@ public class DistributedFileSystem extends FileSystem { }.resolve(this, absF); } + @Override + public FileChecksum getFileChecksum(Path f, final long length) + throws IOException { + statistics.incrementReadOps(1); + Path absF = fixRelativePart(f); + return new FileSystemLinkResolver() { + @Override + public FileChecksum doCall(final Path p) + throws IOException, UnresolvedLinkException { + return dfs.getFileChecksum(getPathName(p), length); + } + + @Override + public FileChecksum next(final FileSystem fs, final Path p) + throws IOException { + if (fs instanceof DistributedFileSystem) { + return ((DistributedFileSystem) fs).getFileChecksum(p, length); + } else { + throw new UnsupportedFileSystemException( + "getFileChecksum(Path, long) is not supported by " + + fs.getClass().getSimpleName()); + } + } + }.resolve(this, absF); + } + @Override public void setPermission(Path p, final FsPermission permission ) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index a0de9fa83ef..a118fcb1ff3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -42,6 +42,7 @@ import java.net.Socket; import java.net.SocketException; import java.net.UnknownHostException; import java.nio.channels.ClosedChannelException; +import java.security.MessageDigest; import java.util.Arrays; import org.apache.commons.logging.Log; @@ -83,6 +84,7 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.DataChecksum; +import com.google.common.base.Preconditions; import com.google.common.net.InetAddresses; import com.google.protobuf.ByteString; @@ -802,7 +804,44 @@ class DataXceiver extends Receiver implements Runnable { IOUtils.closeStream(out); } } - + + private MD5Hash calcPartialBlockChecksum(ExtendedBlock block, + long requestLength, DataChecksum checksum, DataInputStream checksumIn) + throws IOException { + final int bytesPerCRC = checksum.getBytesPerChecksum(); + final int csize = checksum.getChecksumSize(); + final byte[] buffer = new byte[4*1024]; + MessageDigest digester = MD5Hash.getDigester(); + + long remaining = requestLength / bytesPerCRC * csize; + for (int toDigest = 0; remaining > 0; remaining -= toDigest) { + toDigest = checksumIn.read(buffer, 0, + (int) Math.min(remaining, buffer.length)); + if (toDigest < 0) { + break; + } + digester.update(buffer, 0, toDigest); + } + + int partialLength = (int) (requestLength % bytesPerCRC); + if (partialLength > 0) { + byte[] buf = new byte[partialLength]; + final InputStream blockIn = datanode.data.getBlockInputStream(block, + requestLength - partialLength); + try { + // Get the CRC of the partialLength. + IOUtils.readFully(blockIn, buf, 0, partialLength); + } finally { + IOUtils.closeStream(blockIn); + } + checksum.update(buf, 0, partialLength); + byte[] partialCrc = new byte[csize]; + checksum.writeValue(partialCrc, 0, true); + digester.update(partialCrc); + } + return new MD5Hash(digester.digest()); + } + @Override public void blockChecksum(final ExtendedBlock block, final Token blockToken) throws IOException { @@ -810,25 +849,32 @@ class DataXceiver extends Receiver implements Runnable { getOutputStream()); checkAccess(out, true, block, blockToken, Op.BLOCK_CHECKSUM, BlockTokenSecretManager.AccessMode.READ); - updateCurrentThreadName("Reading metadata for block " + block); - final LengthInputStream metadataIn = - datanode.data.getMetaDataInputStream(block); - final DataInputStream checksumIn = new DataInputStream(new BufferedInputStream( - metadataIn, HdfsConstants.IO_FILE_BUFFER_SIZE)); + // client side now can specify a range of the block for checksum + long requestLength = block.getNumBytes(); + Preconditions.checkArgument(requestLength >= 0); + long visibleLength = datanode.data.getReplicaVisibleLength(block); + boolean partialBlk = requestLength < visibleLength; + updateCurrentThreadName("Reading metadata for block " + block); + final LengthInputStream metadataIn = datanode.data + .getMetaDataInputStream(block); + + final DataInputStream checksumIn = new DataInputStream( + new BufferedInputStream(metadataIn, HdfsConstants.IO_FILE_BUFFER_SIZE)); updateCurrentThreadName("Getting checksum for block " + block); try { //read metadata file - final BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn); - final DataChecksum checksum = header.getChecksum(); + final BlockMetadataHeader header = BlockMetadataHeader + .readHeader(checksumIn); + final DataChecksum checksum = header.getChecksum(); + final int csize = checksum.getChecksumSize(); final int bytesPerCRC = checksum.getBytesPerChecksum(); - final long crcPerBlock = checksum.getChecksumSize() > 0 - ? (metadataIn.getLength() - BlockMetadataHeader.getHeaderSize())/checksum.getChecksumSize() - : 0; - - //compute block checksum - final MD5Hash md5 = MD5Hash.digest(checksumIn); + final long crcPerBlock = csize <= 0 ? 0 : + (metadataIn.getLength() - BlockMetadataHeader.getHeaderSize()) / csize; + final MD5Hash md5 = partialBlk && crcPerBlock > 0 ? + calcPartialBlockChecksum(block, requestLength, checksum, checksumIn) + : MD5Hash.digest(checksumIn); if (LOG.isDebugEnabled()) { LOG.debug("block=" + block + ", bytesPerCRC=" + bytesPerCRC + ", crcPerBlock=" + crcPerBlock + ", md5=" + md5); @@ -841,8 +887,7 @@ class DataXceiver extends Receiver implements Runnable { .setBytesPerCrc(bytesPerCRC) .setCrcPerBlock(crcPerBlock) .setMd5(ByteString.copyFrom(md5.getDigest())) - .setCrcType(PBHelper.convert(checksum.getChecksumType())) - ) + .setCrcType(PBHelper.convert(checksum.getChecksumType()))) .build() .writeDelimitedTo(out); out.flush(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java index fb144d39b82..83de6ebe41b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java @@ -74,7 +74,6 @@ import org.apache.hadoop.hdfs.web.resources.PutOpParam; import org.apache.hadoop.hdfs.web.resources.ReplicationParam; import org.apache.hadoop.hdfs.web.resources.UriFsPathParam; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.Text; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -452,7 +451,7 @@ public class DatanodeWebHdfsMethods { MD5MD5CRC32FileChecksum checksum = null; DFSClient dfsclient = newDfsClient(nnId, conf); try { - checksum = dfsclient.getFileChecksum(fullpath); + checksum = dfsclient.getFileChecksum(fullpath, Long.MAX_VALUE); dfsclient.close(); dfsclient = null; } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java new file mode 100644 index 00000000000..0e56ba7f05e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.Path; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestGetFileChecksum { + private static final int BLOCKSIZE = 1024; + private static final short REPLICATION = 3; + + private Configuration conf; + private MiniDFSCluster cluster; + private DistributedFileSystem dfs; + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) + .build(); + cluster.waitActive(); + dfs = cluster.getFileSystem(); + } + + @After + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + } + + public void testGetFileChecksum(final Path foo, final int appendLength) + throws Exception { + final int appendRounds = 16; + FileChecksum[] fc = new FileChecksum[appendRounds + 1]; + DFSTestUtil.createFile(dfs, foo, appendLength, REPLICATION, 0L); + fc[0] = dfs.getFileChecksum(foo); + for (int i = 0; i < appendRounds; i++) { + DFSTestUtil.appendFile(dfs, foo, appendLength); + fc[i + 1] = dfs.getFileChecksum(foo); + } + + for (int i = 0; i < appendRounds + 1; i++) { + FileChecksum checksum = dfs.getFileChecksum(foo, appendLength * (i+1)); + Assert.assertTrue(checksum.equals(fc[i])); + } + } + + @Test + public void testGetFileChecksum() throws Exception { + testGetFileChecksum(new Path("/foo"), BLOCKSIZE / 4); + testGetFileChecksum(new Path("/bar"), BLOCKSIZE / 4 - 1); + } +} diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ce257506ff9..82e90984420 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -202,6 +202,8 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5809. Enhance distcp to support preserving HDFS ACLs. (cnauroth) + MAPREDUCE-5899. Support incremental data copy in DistCp. (jing9) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java index 695d8bde394..5fa26da6135 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java @@ -50,6 +50,7 @@ public class DistCpConstants { public static final String CONF_LABEL_COPY_STRATEGY = "distcp.copy.strategy"; public static final String CONF_LABEL_SKIP_CRC = "distcp.skip.crc"; public static final String CONF_LABEL_OVERWRITE = "distcp.copy.overwrite"; + public static final String CONF_LABEL_APPEND = "distcp.copy.append"; public static final String CONF_LABEL_BANDWIDTH_MB = "distcp.map.bandwidth.mb"; public static final String CONF_LABEL_MAX_CHUNKS_TOLERABLE = diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java index 1639c1d8373..bfaba966be7 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java @@ -138,6 +138,10 @@ public enum DistCpOptionSwitch { new Option("overwrite", false, "Choose to overwrite target files " + "unconditionally, even if they exist.")), + APPEND(DistCpConstants.CONF_LABEL_APPEND, + new Option("append", false, + "Reuse existing data in target files and append new data to them if possible")), + /** * Should DisctpExecution be blocking */ diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java index 5906266fb78..2d19c6afc18 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java @@ -39,6 +39,7 @@ public class DistCpOptions { private boolean deleteMissing = false; private boolean ignoreFailures = false; private boolean overwrite = false; + private boolean append = false; private boolean skipCRC = false; private boolean blocking = true; @@ -244,6 +245,22 @@ public class DistCpOptions { this.overwrite = overwrite; } + /** + * @return whether we can append new data to target files + */ + public boolean shouldAppend() { + return append; + } + + /** + * Set if we want to append new data to target files. This is valid only with + * update option and CRC is not skipped. + */ + public void setAppend(boolean append) { + validate(DistCpOptionSwitch.APPEND, append); + this.append = append; + } + /** * Should CRC/checksum check be skipped while checking files are identical * @@ -472,6 +489,7 @@ public class DistCpOptions { value : this.atomicCommit); boolean skipCRC = (option == DistCpOptionSwitch.SKIP_CRC ? value : this.skipCRC); + boolean append = (option == DistCpOptionSwitch.APPEND ? value : this.append); if (syncFolder && atomicCommit) { throw new IllegalArgumentException("Atomic commit can't be used with " + @@ -492,6 +510,14 @@ public class DistCpOptions { throw new IllegalArgumentException("Skip CRC is valid only with update options"); } + if (!syncFolder && append) { + throw new IllegalArgumentException( + "Append is valid only with update options"); + } + if (skipCRC && append) { + throw new IllegalArgumentException( + "Append is disallowed when skipping CRC"); + } } /** @@ -510,6 +536,8 @@ public class DistCpOptions { String.valueOf(deleteMissing)); DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.OVERWRITE, String.valueOf(overwrite)); + DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.APPEND, + String.valueOf(append)); DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.SKIP_CRC, String.valueOf(skipCRC)); DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH, diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java index d36311d17b8..09e85505227 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java @@ -140,6 +140,10 @@ public class OptionsParser { option.setOverwrite(true); } + if (command.hasOption(DistCpOptionSwitch.APPEND.getSwitch())) { + option.setAppend(true); + } + if (command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch())) { option.setDeleteMissing(true); } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java index caf4057c6c2..02337f78bbc 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java @@ -18,13 +18,20 @@ package org.apache.hadoop.tools.mapred; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.EnumSet; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.Mapper; @@ -36,11 +43,6 @@ import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.util.DistCpUtils; import org.apache.hadoop.util.StringUtils; -import java.io.*; -import java.util.EnumSet; -import java.util.Arrays; -import java.util.List; - /** * Mapper class that executes the DistCp copy operation. * Implements the o.a.h.mapreduce.Mapper<> interface. @@ -62,6 +64,15 @@ public class CopyMapper extends Mapper BYTESSKIPPED, // Number of bytes that were skipped from copy. } + /** + * Indicate the action for each file + */ + static enum FileAction { + SKIP, // Skip copying the file since it's already in the target FS + APPEND, // Only need to append new data to the file in the target FS + OVERWRITE, // Overwrite the whole file + } + private static Log LOG = LogFactory.getLog(CopyMapper.class); private Configuration conf; @@ -70,6 +81,7 @@ public class CopyMapper extends Mapper private boolean ignoreFailures = false; private boolean skipCrc = false; private boolean overWrite = false; + private boolean append = false; private EnumSet preserve = EnumSet.noneOf(FileAttribute.class); private FileSystem targetFS = null; @@ -90,6 +102,7 @@ public class CopyMapper extends Mapper ignoreFailures = conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false); skipCrc = conf.getBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), false); overWrite = conf.getBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), false); + append = conf.getBoolean(DistCpOptionSwitch.APPEND.getConfigLabel(), false); preserve = DistCpUtils.unpackAttributes(conf.get(DistCpOptionSwitch. PRESERVE_STATUS.getConfigLabel())); @@ -224,20 +237,19 @@ public class CopyMapper extends Mapper return; } - if (skipFile(sourceFS, sourceCurrStatus, target)) { + FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target); + if (action == FileAction.SKIP) { LOG.info("Skipping copy of " + sourceCurrStatus.getPath() + " to " + target); updateSkipCounters(context, sourceCurrStatus); context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath())); - } - else { + } else { copyFileWithRetry(description, sourceCurrStatus, target, context, - fileAttributes); + action, fileAttributes); } DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus, fileAttributes); - } catch (IOException exception) { handleFailures(exception, sourceFileStatus, target, context); } @@ -254,14 +266,14 @@ public class CopyMapper extends Mapper return DistCpUtils.unpackAttributes(attributeString); } - private void copyFileWithRetry(String description, FileStatus sourceFileStatus, - Path target, Context context, - EnumSet fileAttributes) throws IOException { - + private void copyFileWithRetry(String description, + FileStatus sourceFileStatus, Path target, Context context, + FileAction action, EnumSet fileAttributes) + throws IOException { long bytesCopied; try { - bytesCopied = (Long)new RetriableFileCopyCommand(skipCrc, description) - .execute(sourceFileStatus, target, context, fileAttributes); + bytesCopied = (Long) new RetriableFileCopyCommand(skipCrc, description, + action).execute(sourceFileStatus, target, context, fileAttributes); } catch (Exception e) { context.setStatus("Copy Failure: " + sourceFileStatus.getPath()); throw new IOException("File copy failed: " + sourceFileStatus.getPath() + @@ -311,25 +323,48 @@ public class CopyMapper extends Mapper context.getCounter(counter).increment(value); } - private boolean skipFile(FileSystem sourceFS, FileStatus source, Path target) - throws IOException { - return targetFS.exists(target) - && !overWrite - && !mustUpdate(sourceFS, source, target); + private FileAction checkUpdate(FileSystem sourceFS, FileStatus source, + Path target) throws IOException { + final FileStatus targetFileStatus; + try { + targetFileStatus = targetFS.getFileStatus(target); + } catch (FileNotFoundException e) { + return FileAction.OVERWRITE; + } + if (targetFileStatus != null && !overWrite) { + if (canSkip(sourceFS, source, targetFileStatus)) { + return FileAction.SKIP; + } else if (append) { + long targetLen = targetFileStatus.getLen(); + if (targetLen < source.getLen()) { + FileChecksum sourceChecksum = sourceFS.getFileChecksum( + source.getPath(), targetLen); + if (sourceChecksum != null + && sourceChecksum.equals(targetFS.getFileChecksum(target))) { + // We require that the checksum is not null. Thus currently only + // DistributedFileSystem is supported + return FileAction.APPEND; + } + } + } + } + return FileAction.OVERWRITE; } - private boolean mustUpdate(FileSystem sourceFS, FileStatus source, Path target) - throws IOException { - final FileStatus targetFileStatus = targetFS.getFileStatus(target); - - return syncFolders - && ( - targetFileStatus.getLen() != source.getLen() - || (!skipCrc && - !DistCpUtils.checksumsAreEqual(sourceFS, - source.getPath(), null, targetFS, target)) - || (source.getBlockSize() != targetFileStatus.getBlockSize() && - preserve.contains(FileAttribute.BLOCKSIZE)) - ); + private boolean canSkip(FileSystem sourceFS, FileStatus source, + FileStatus target) throws IOException { + if (!syncFolders) { + return true; + } + boolean sameLength = target.getLen() == source.getLen(); + boolean sameBlockSize = source.getBlockSize() == target.getBlockSize() + || !preserve.contains(FileAttribute.BLOCKSIZE); + if (sameLength && sameBlockSize) { + return skipCrc || + DistCpUtils.checksumsAreEqual(sourceFS, source.getPath(), null, + targetFS, target.getPath()); + } else { + return false; + } } } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java index 1b735262581..1d6115685e5 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java @@ -18,10 +18,8 @@ package org.apache.hadoop.tools.mapred; -import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.util.EnumSet; @@ -29,6 +27,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -39,6 +39,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.tools.DistCpConstants; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; +import org.apache.hadoop.tools.mapred.CopyMapper.FileAction; import org.apache.hadoop.tools.util.DistCpUtils; import org.apache.hadoop.tools.util.RetriableCommand; import org.apache.hadoop.tools.util.ThrottledInputStream; @@ -54,13 +55,15 @@ public class RetriableFileCopyCommand extends RetriableCommand { private static Log LOG = LogFactory.getLog(RetriableFileCopyCommand.class); private static int BUFFER_SIZE = 8 * 1024; private boolean skipCrc = false; + private FileAction action; /** * Constructor, taking a description of the action. * @param description Verbose description of the copy operation. */ - public RetriableFileCopyCommand(String description) { + public RetriableFileCopyCommand(String description, FileAction action) { super(description); + this.action = action; } /** @@ -68,9 +71,11 @@ public class RetriableFileCopyCommand extends RetriableCommand { * * @param skipCrc Whether to skip the crc check. * @param description A verbose description of the copy operation. + * @param action We should overwrite the target file or append new data to it. */ - public RetriableFileCopyCommand(boolean skipCrc, String description) { - this(description); + public RetriableFileCopyCommand(boolean skipCrc, String description, + FileAction action) { + this(description, action); this.skipCrc = skipCrc; } @@ -96,18 +101,17 @@ public class RetriableFileCopyCommand extends RetriableCommand { } private long doCopy(FileStatus sourceFileStatus, Path target, - Mapper.Context context, - EnumSet fileAttributes) - throws IOException { - - Path tmpTargetPath = getTmpFile(target, context); + Mapper.Context context, EnumSet fileAttributes) + throws IOException { + final boolean toAppend = action == FileAction.APPEND; + Path targetPath = toAppend ? target : getTmpFile(target, context); final Configuration configuration = context.getConfiguration(); FileSystem targetFS = target.getFileSystem(configuration); try { if (LOG.isDebugEnabled()) { LOG.debug("Copying " + sourceFileStatus.getPath() + " to " + target); - LOG.debug("Tmp-file path: " + tmpTargetPath); + LOG.debug("Target file path: " + targetPath); } final Path sourcePath = sourceFileStatus.getPath(); final FileSystem sourceFS = sourcePath.getFileSystem(configuration); @@ -115,22 +119,31 @@ public class RetriableFileCopyCommand extends RetriableCommand { .contains(FileAttribute.CHECKSUMTYPE) ? sourceFS .getFileChecksum(sourcePath) : null; - long bytesRead = copyToTmpFile(tmpTargetPath, targetFS, sourceFileStatus, - context, fileAttributes, sourceChecksum); + final long offset = action == FileAction.APPEND ? targetFS.getFileStatus( + target).getLen() : 0; + long bytesRead = copyToFile(targetPath, targetFS, sourceFileStatus, + offset, context, fileAttributes, sourceChecksum); - compareFileLengths(sourceFileStatus, tmpTargetPath, configuration, - bytesRead); + compareFileLengths(sourceFileStatus, targetPath, configuration, bytesRead + + offset); //At this point, src&dest lengths are same. if length==0, we skip checksum if ((bytesRead != 0) && (!skipCrc)) { compareCheckSums(sourceFS, sourceFileStatus.getPath(), sourceChecksum, - targetFS, tmpTargetPath); + targetFS, targetPath); + } + // it's not append case, thus we first write to a temporary file, rename + // it to the target path. + if (!toAppend) { + promoteTmpToTarget(targetPath, target, targetFS); } - promoteTmpToTarget(tmpTargetPath, target, targetFS); return bytesRead; - } finally { - if (targetFS.exists(tmpTargetPath)) - targetFS.delete(tmpTargetPath, false); + // note that for append case, it is possible that we append partial data + // and then fail. In that case, for the next retry, we either reuse the + // partial appended data if it is good or we overwrite the whole file + if (!toAppend && targetFS.exists(targetPath)) { + targetFS.delete(targetPath, false); + } } } @@ -147,29 +160,37 @@ public class RetriableFileCopyCommand extends RetriableCommand { return null; } - private long copyToTmpFile(Path tmpTargetPath, FileSystem targetFS, - FileStatus sourceFileStatus, Mapper.Context context, + private long copyToFile(Path targetPath, FileSystem targetFS, + FileStatus sourceFileStatus, long sourceOffset, Mapper.Context context, EnumSet fileAttributes, final FileChecksum sourceChecksum) throws IOException { FsPermission permission = FsPermission.getFileDefault().applyUMask( FsPermission.getUMask(targetFS.getConf())); - OutputStream outStream = new BufferedOutputStream( - targetFS.create(tmpTargetPath, permission, - EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), BUFFER_SIZE, - getReplicationFactor(fileAttributes, sourceFileStatus, targetFS, - tmpTargetPath), - getBlockSize(fileAttributes, sourceFileStatus, targetFS, - tmpTargetPath), - context, getChecksumOpt(fileAttributes, sourceChecksum))); - return copyBytes(sourceFileStatus, outStream, BUFFER_SIZE, context); + final OutputStream outStream; + if (action == FileAction.OVERWRITE) { + final short repl = getReplicationFactor(fileAttributes, sourceFileStatus, + targetFS, targetPath); + final long blockSize = getBlockSize(fileAttributes, sourceFileStatus, + targetFS, targetPath); + FSDataOutputStream out = targetFS.create(targetPath, permission, + EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), + BUFFER_SIZE, repl, blockSize, context, + getChecksumOpt(fileAttributes, sourceChecksum)); + outStream = new BufferedOutputStream(out); + } else { + outStream = new BufferedOutputStream(targetFS.append(targetPath, + BUFFER_SIZE)); + } + return copyBytes(sourceFileStatus, sourceOffset, outStream, BUFFER_SIZE, + context); } private void compareFileLengths(FileStatus sourceFileStatus, Path target, - Configuration configuration, long bytesRead) + Configuration configuration, long targetLen) throws IOException { final Path sourcePath = sourceFileStatus.getPath(); FileSystem fs = sourcePath.getFileSystem(configuration); - if (fs.getFileStatus(sourcePath).getLen() != bytesRead) + if (fs.getFileStatus(sourcePath).getLen() != targetLen) throw new IOException("Mismatch in length of source:" + sourcePath + " and target:" + target); } @@ -215,8 +236,8 @@ public class RetriableFileCopyCommand extends RetriableCommand { } @VisibleForTesting - long copyBytes(FileStatus sourceFileStatus, OutputStream outStream, - int bufferSize, Mapper.Context context) + long copyBytes(FileStatus sourceFileStatus, long sourceOffset, + OutputStream outStream, int bufferSize, Mapper.Context context) throws IOException { Path source = sourceFileStatus.getPath(); byte buf[] = new byte[bufferSize]; @@ -225,19 +246,21 @@ public class RetriableFileCopyCommand extends RetriableCommand { try { inStream = getInputStream(source, context.getConfiguration()); - int bytesRead = readBytes(inStream, buf); + int bytesRead = readBytes(inStream, buf, sourceOffset); while (bytesRead >= 0) { totalBytesRead += bytesRead; + if (action == FileAction.APPEND) { + sourceOffset += bytesRead; + } outStream.write(buf, 0, bytesRead); updateContextStatus(totalBytesRead, context, sourceFileStatus); - bytesRead = inStream.read(buf); + bytesRead = readBytes(inStream, buf, sourceOffset); } outStream.close(); outStream = null; } finally { IOUtils.cleanup(LOG, outStream, inStream); } - return totalBytesRead; } @@ -254,24 +277,27 @@ public class RetriableFileCopyCommand extends RetriableCommand { context.setStatus(message.toString()); } - private static int readBytes(InputStream inStream, byte buf[]) - throws IOException { + private static int readBytes(ThrottledInputStream inStream, byte buf[], + long position) throws IOException { try { - return inStream.read(buf); - } - catch (IOException e) { + if (position == 0) { + return inStream.read(buf); + } else { + return inStream.read(position, buf, 0, buf.length); + } + } catch (IOException e) { throw new CopyReadException(e); } } - private static ThrottledInputStream getInputStream(Path path, Configuration conf) - throws IOException { + private static ThrottledInputStream getInputStream(Path path, + Configuration conf) throws IOException { try { FileSystem fs = path.getFileSystem(conf); long bandwidthMB = conf.getInt(DistCpConstants.CONF_LABEL_BANDWIDTH_MB, DistCpConstants.DEFAULT_BANDWIDTH_MB); - return new ThrottledInputStream(new BufferedInputStream(fs.open(path)), - bandwidthMB * 1024 * 1024); + FSDataInputStream in = fs.open(path); + return new ThrottledInputStream(in, bandwidthMB * 1024 * 1024); } catch (IOException e) { throw new CopyReadException(e); diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java index 75ae86ad054..f6fe11847a2 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java @@ -21,6 +21,11 @@ package org.apache.hadoop.tools.util; import java.io.IOException; import java.io.InputStream; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.PositionedReadable; + +import com.google.common.base.Preconditions; + /** * The ThrottleInputStream provides bandwidth throttling on a specified * InputStream. It is implemented as a wrapper on top of another InputStream @@ -90,6 +95,25 @@ public class ThrottledInputStream extends InputStream { return readLen; } + /** + * Read bytes starting from the specified position. This requires rawStream is + * an instance of {@link PositionedReadable}. + */ + public int read(long position, byte[] buffer, int offset, int length) + throws IOException { + if (!(rawStream instanceof PositionedReadable)) { + throw new UnsupportedOperationException( + "positioned read is not supported by the internal stream"); + } + throttle(); + int readLen = ((PositionedReadable) rawStream).read(position, buffer, + offset, length); + if (readLen != -1) { + bytesRead += readLen; + } + return readLen; + } + private void throttle() throws IOException { if (getBytesPerSec() > maxBytesPerSec) { try { diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java index 296e994f930..8486aa1fcea 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java @@ -18,9 +18,12 @@ package org.apache.hadoop.tools; +import static org.junit.Assert.fail; + import org.junit.Assert; import org.junit.Test; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.tools.DistCpOptions.*; import org.apache.hadoop.conf.Configuration; @@ -554,4 +557,45 @@ public class TestOptionsParser { Assert.assertEquals(conf.get(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel()), "U"); Assert.assertEquals(conf.getInt(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 11); } + + @Test + public void testAppendOption() { + Configuration conf = new Configuration(); + Assert.assertFalse(conf.getBoolean( + DistCpOptionSwitch.APPEND.getConfigLabel(), false)); + Assert.assertFalse(conf.getBoolean( + DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); + + DistCpOptions options = OptionsParser.parse(new String[] { "-update", + "-append", "hdfs://localhost:8020/source/first", + "hdfs://localhost:8020/target/" }); + options.appendToConf(conf); + Assert.assertTrue(conf.getBoolean( + DistCpOptionSwitch.APPEND.getConfigLabel(), false)); + Assert.assertTrue(conf.getBoolean( + DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); + + // make sure -append is only valid when -update is specified + try { + options = OptionsParser.parse(new String[] { "-append", + "hdfs://localhost:8020/source/first", + "hdfs://localhost:8020/target/" }); + fail("Append should fail if update option is not specified"); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "Append is valid only with update options", e); + } + + // make sure -append is invalid when skipCrc is specified + try { + options = OptionsParser.parse(new String[] { + "-append", "-update", "-skipcrccheck", + "hdfs://localhost:8020/source/first", + "hdfs://localhost:8020/target/" }); + fail("Append should fail if skipCrc option is specified"); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "Append is disallowed when skipping CRC", e); + } + } } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java index 7eb1b6801e0..2f16682fcd0 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java @@ -25,11 +25,13 @@ import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options.ChecksumOpt; @@ -118,6 +120,16 @@ public class TestCopyMapper { touchFile(SOURCE_PATH + "/7/8/9"); } + private static void appendSourceData() throws Exception { + FileSystem fs = cluster.getFileSystem(); + for (Path source : pathList) { + if (fs.getFileStatus(source).isFile()) { + // append 2048 bytes per file + appendFile(source, DEFAULT_FILE_SIZE * 2); + } + } + } + private static void createSourceDataWithDifferentBlockSize() throws Exception { mkdirs(SOURCE_PATH + "/1"); mkdirs(SOURCE_PATH + "/2"); @@ -201,85 +213,132 @@ public class TestCopyMapper { } } + /** + * Append specified length of bytes to a given file + */ + private static void appendFile(Path p, int length) throws IOException { + byte[] toAppend = new byte[length]; + Random random = new Random(); + random.nextBytes(toAppend); + FSDataOutputStream out = cluster.getFileSystem().append(p); + try { + out.write(toAppend); + } finally { + IOUtils.closeStream(out); + } + } + @Test public void testCopyWithDifferentChecksumType() throws Exception { testCopy(true); } @Test(timeout=40000) - public void testRun() { + public void testRun() throws Exception { testCopy(false); } - private void testCopy(boolean preserveChecksum) { - try { - deleteState(); - if (preserveChecksum) { - createSourceDataWithDifferentChecksumType(); - } else { - createSourceData(); - } + @Test + public void testCopyWithAppend() throws Exception { + final FileSystem fs = cluster.getFileSystem(); + // do the first distcp + testCopy(false); + // start appending data to source + appendSourceData(); - FileSystem fs = cluster.getFileSystem(); - CopyMapper copyMapper = new CopyMapper(); - StubContext stubContext = new StubContext(getConfiguration(), null, 0); - Mapper.Context context - = stubContext.getContext(); - - Configuration configuration = context.getConfiguration(); - EnumSet fileAttributes - = EnumSet.of(DistCpOptions.FileAttribute.REPLICATION); - if (preserveChecksum) { - fileAttributes.add(DistCpOptions.FileAttribute.CHECKSUMTYPE); - } - configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(), - DistCpUtils.packAttributes(fileAttributes)); - - copyMapper.setup(context); - - for (Path path: pathList) { - copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), - new CopyListingFileStatus(fs.getFileStatus(path)), context); - } - - // Check that the maps worked. - for (Path path : pathList) { - final Path targetPath = new Path(path.toString() - .replaceAll(SOURCE_PATH, TARGET_PATH)); - Assert.assertTrue(fs.exists(targetPath)); - Assert.assertTrue(fs.isFile(targetPath) == fs.isFile(path)); - FileStatus sourceStatus = fs.getFileStatus(path); - FileStatus targetStatus = fs.getFileStatus(targetPath); - Assert.assertEquals(sourceStatus.getReplication(), - targetStatus.getReplication()); - if (preserveChecksum) { - Assert.assertEquals(sourceStatus.getBlockSize(), - targetStatus.getBlockSize()); - } - Assert.assertTrue(!fs.isFile(targetPath) - || fs.getFileChecksum(targetPath).equals(fs.getFileChecksum(path))); - } - - Assert.assertEquals(pathList.size(), - stubContext.getReporter().getCounter(CopyMapper.Counter.COPY).getValue()); - if (!preserveChecksum) { - Assert.assertEquals(nFiles * DEFAULT_FILE_SIZE, stubContext - .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED) - .getValue()); - } else { - Assert.assertEquals(nFiles * NON_DEFAULT_BLOCK_SIZE * 2, stubContext - .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED) - .getValue()); - } - - testCopyingExistingFiles(fs, copyMapper, context); - for (Text value : stubContext.getWriter().values()) { - Assert.assertTrue(value.toString() + " is not skipped", value.toString().startsWith("SKIP:")); - } + // do the distcp again with -update and -append option + CopyMapper copyMapper = new CopyMapper(); + StubContext stubContext = new StubContext(getConfiguration(), null, 0); + Mapper.Context context = + stubContext.getContext(); + // Enable append + context.getConfiguration().setBoolean( + DistCpOptionSwitch.APPEND.getConfigLabel(), true); + copyMapper.setup(context); + for (Path path: pathList) { + copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), + new CopyListingFileStatus(cluster.getFileSystem().getFileStatus( + path)), context); } - catch (Exception e) { - LOG.error("Unexpected exception: ", e); - Assert.assertTrue(false); + + verifyCopy(fs, false); + // verify that we only copied new appended data + Assert.assertEquals(nFiles * DEFAULT_FILE_SIZE * 2, stubContext + .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED) + .getValue()); + Assert.assertEquals(pathList.size(), stubContext.getReporter(). + getCounter(CopyMapper.Counter.COPY).getValue()); + } + + private void testCopy(boolean preserveChecksum) throws Exception { + deleteState(); + if (preserveChecksum) { + createSourceDataWithDifferentChecksumType(); + } else { + createSourceData(); + } + + FileSystem fs = cluster.getFileSystem(); + CopyMapper copyMapper = new CopyMapper(); + StubContext stubContext = new StubContext(getConfiguration(), null, 0); + Mapper.Context context + = stubContext.getContext(); + + Configuration configuration = context.getConfiguration(); + EnumSet fileAttributes + = EnumSet.of(DistCpOptions.FileAttribute.REPLICATION); + if (preserveChecksum) { + fileAttributes.add(DistCpOptions.FileAttribute.CHECKSUMTYPE); + } + configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(), + DistCpUtils.packAttributes(fileAttributes)); + + copyMapper.setup(context); + + for (Path path: pathList) { + copyMapper.map( + new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)), + new CopyListingFileStatus(fs.getFileStatus(path)), context); + } + + // Check that the maps worked. + verifyCopy(fs, preserveChecksum); + Assert.assertEquals(pathList.size(), stubContext.getReporter() + .getCounter(CopyMapper.Counter.COPY).getValue()); + if (!preserveChecksum) { + Assert.assertEquals(nFiles * DEFAULT_FILE_SIZE, stubContext + .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED) + .getValue()); + } else { + Assert.assertEquals(nFiles * NON_DEFAULT_BLOCK_SIZE * 2, stubContext + .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED) + .getValue()); + } + + testCopyingExistingFiles(fs, copyMapper, context); + for (Text value : stubContext.getWriter().values()) { + Assert.assertTrue(value.toString() + " is not skipped", value + .toString().startsWith("SKIP:")); + } + } + + private void verifyCopy(FileSystem fs, boolean preserveChecksum) + throws Exception { + for (Path path : pathList) { + final Path targetPath = new Path(path.toString().replaceAll(SOURCE_PATH, + TARGET_PATH)); + Assert.assertTrue(fs.exists(targetPath)); + Assert.assertTrue(fs.isFile(targetPath) == fs.isFile(path)); + FileStatus sourceStatus = fs.getFileStatus(path); + FileStatus targetStatus = fs.getFileStatus(targetPath); + Assert.assertEquals(sourceStatus.getReplication(), + targetStatus.getReplication()); + if (preserveChecksum) { + Assert.assertEquals(sourceStatus.getBlockSize(), + targetStatus.getBlockSize()); + } + Assert.assertTrue(!fs.isFile(targetPath) + || fs.getFileChecksum(targetPath).equals(fs.getFileChecksum(path))); } } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java index c5ec513bec5..f1b8532b74c 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.*; +import org.apache.hadoop.tools.mapred.CopyMapper.FileAction; import org.junit.Test; import static org.junit.Assert.*; import static org.mockito.Mockito.*; @@ -48,8 +49,8 @@ public class TestRetriableFileCopyCommand { Exception actualEx = null; try { - new RetriableFileCopyCommand("testFailOnCloseError") - .copyBytes(stat, out, 512, context); + new RetriableFileCopyCommand("testFailOnCloseError", FileAction.OVERWRITE) + .copyBytes(stat, 0, out, 512, context); } catch (Exception e) { actualEx = e; } From 3c4c16a4f7ee1f8bb02367d734281580ad0739ce Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 22 May 2014 20:41:21 +0000 Subject: [PATCH 02/34] HADOOP-10618. Remove SingleNodeSetup.apt.vm (Contributed by Akira Ajisaka) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1596964 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../src/site/apt/SingleNodeSetup.apt.vm | 209 +----------------- 2 files changed, 6 insertions(+), 206 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 93d6431bd26..aa93ffc58f9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -413,6 +413,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10614. CBZip2InputStream is not threadsafe (Xiangrui Meng via Sandy Ryza) + HADOOP-10618. Remove SingleNodeSetup.apt.vm. (Akira Ajisaka via + Arpit Agarwal) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm index ebfb1466bdb..eb0c8015116 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm @@ -18,210 +18,7 @@ Single Node Setup -%{toc|section=1|fromDepth=0} + This page will be removed in the next major release. -* Purpose - - This document describes how to set up and configure a single-node - Hadoop installation so that you can quickly perform simple operations - using Hadoop MapReduce and the Hadoop Distributed File System (HDFS). - -* Prerequisites - -** Supported Platforms - - * GNU/Linux is supported as a development and production platform. - Hadoop has been demonstrated on GNU/Linux clusters with 2000 nodes. - - * Windows is also a supported platform. - -** Required Software - - Required software for Linux and Windows include: - - [[1]] Java^TM 1.6.x, preferably from Sun, must be installed. - - [[2]] ssh must be installed and sshd must be running to use the Hadoop - scripts that manage remote Hadoop daemons. - -** Installing Software - - If your cluster doesn't have the requisite software you will need to - install it. - - For example on Ubuntu Linux: - ----- - $ sudo apt-get install ssh - $ sudo apt-get install rsync ----- - -* Download - - To get a Hadoop distribution, download a recent stable release from one - of the Apache Download Mirrors. - -* Prepare to Start the Hadoop Cluster - - Unpack the downloaded Hadoop distribution. In the distribution, edit - the file <<>> to define at least <<>> to be the root - of your Java installation. - - Try the following command: - ----- - $ bin/hadoop ----- - - This will display the usage documentation for the hadoop script. - - Now you are ready to start your Hadoop cluster in one of the three - supported modes: - - * Local (Standalone) Mode - - * Pseudo-Distributed Mode - - * Fully-Distributed Mode - -* Standalone Operation - - By default, Hadoop is configured to run in a non-distributed mode, as a - single Java process. This is useful for debugging. - - The following example copies the unpacked conf directory to use as - input and then finds and displays every match of the given regular - expression. Output is written to the given output directory. - ----- - $ mkdir input - $ cp conf/*.xml input - $ bin/hadoop jar hadoop-*-examples.jar grep input output 'dfs[a-z.]+' - $ cat output/* ---- - -* Pseudo-Distributed Operation - - Hadoop can also be run on a single-node in a pseudo-distributed mode - where each Hadoop daemon runs in a separate Java process. - -** Configuration - - Use the following: - - conf/core-site.xml: - ----- - - - fs.defaultFS - hdfs://localhost:9000 - - ----- - - conf/hdfs-site.xml: - ----- - - - dfs.replication - 1 - - ----- - - conf/mapred-site.xml: - ----- - - - mapred.job.tracker - localhost:9001 - - ----- - -** Setup passphraseless ssh - - Now check that you can ssh to the localhost without a passphrase: - ----- - $ ssh localhost ----- - - If you cannot ssh to localhost without a passphrase, execute the - following commands: - ----- - $ ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa - $ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys ----- - -** Execution - - Format a new distributed-filesystem: - ----- - $ bin/hadoop namenode -format ----- - - Start the hadoop daemons: - ----- - $ bin/start-all.sh ----- - - The hadoop daemon log output is written to the <<<${HADOOP_LOG_DIR}>>> - directory (defaults to <<<${HADOOP_PREFIX}/logs>>>). - - Browse the web interface for the NameNode and the JobTracker; by - default they are available at: - - * NameNode - <<>> - - * JobTracker - <<>> - - Copy the input files into the distributed filesystem: - ----- - $ bin/hadoop fs -put conf input ----- - - Run some of the examples provided: - ----- - $ bin/hadoop jar hadoop-*-examples.jar grep input output 'dfs[a-z.]+' ----- - - Examine the output files: - - Copy the output files from the distributed filesystem to the local - filesytem and examine them: - ----- - $ bin/hadoop fs -get output output - $ cat output/* ----- - - or - - View the output files on the distributed filesystem: - ----- - $ bin/hadoop fs -cat output/* ----- - - When you're done, stop the daemons with: - ----- - $ bin/stop-all.sh ----- - -* Fully-Distributed Operation - - For information on setting up fully-distributed, non-trivial clusters - see {{{./ClusterSetup.html}Cluster Setup}}. - - Java and JNI are trademarks or registered trademarks of Sun - Microsystems, Inc. in the United States and other countries. + See {{{./SingleCluster.html}Single Cluster Setup}} to set up and configure a + single-node Hadoop installation. From 1ba203e3e0ead72afd9c3351827f44ff2d51b974 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 22 May 2014 20:44:30 +0000 Subject: [PATCH 03/34] HDFS-6435. Add support for specifying a static uid/gid mapping for the NFS gateway. (atm via wang) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1596966 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/nfs/nfs3/IdUserGroup.java | 133 +++++++++++++++++- .../hadoop/nfs/nfs3/TestIdUserGroup.java | 96 ++++++++++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../src/site/apt/HdfsNfsGateway.apt.vm | 15 +- 4 files changed, 234 insertions(+), 13 deletions(-) diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java index 1ed3d3f8bde..10930b90d88 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java @@ -18,8 +18,14 @@ package org.apache.hadoop.nfs.nfs3; import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -44,6 +50,17 @@ public class IdUserGroup { static final String MAC_GET_ALL_USERS_CMD = "dscl . -list /Users UniqueID"; static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID"; + // Used for finding the configured static mapping file. + static final String NFS_STATIC_MAPPING_FILE_KEY = "dfs.nfs.static.mapping.file"; + private static final String NFS_STATIC_MAPPING_FILE_DEFAULT = "/etc/nfs.map"; + private final File staticMappingFile; + + // Used for parsing the static mapping file. + private static final Pattern EMPTY_LINE = Pattern.compile("^\\s*$"); + private static final Pattern COMMENT_LINE = Pattern.compile("^\\s*#.*$"); + private static final Pattern MAPPING_LINE = + Pattern.compile("^(uid|gid)\\s+(\\d+)\\s+(\\d+)\\s*(#.*)?$"); + // Do update every 15 minutes by default final static long TIMEOUT_DEFAULT = 15 * 60 * 1000; // ms final static long TIMEOUT_MIN = 1 * 60 * 1000; // ms @@ -58,6 +75,7 @@ public class IdUserGroup { public IdUserGroup() throws IOException { timeout = TIMEOUT_DEFAULT; + staticMappingFile = new File(NFS_STATIC_MAPPING_FILE_DEFAULT); updateMaps(); } @@ -71,6 +89,11 @@ public class IdUserGroup { } else { timeout = updateTime; } + + String staticFilePath = conf.get(NFS_STATIC_MAPPING_FILE_KEY, + NFS_STATIC_MAPPING_FILE_DEFAULT); + staticMappingFile = new File(staticFilePath); + updateMaps(); } @@ -137,7 +160,8 @@ public class IdUserGroup { */ @VisibleForTesting public static void updateMapInternal(BiMap map, String mapName, - String command, String regex) throws IOException { + String command, String regex, Map staticMapping) + throws IOException { BufferedReader br = null; try { Process process = Runtime.getRuntime().exec( @@ -151,7 +175,7 @@ public class IdUserGroup { } LOG.debug("add to " + mapName + "map:" + nameId[0] + " id:" + nameId[1]); // HDFS can't differentiate duplicate names with simple authentication - final Integer key = parseId(nameId[1]); + final Integer key = staticMapping.get(parseId(nameId[1])); final String value = nameId[0]; if (map.containsKey(key)) { final String prevValue = map.get(key); @@ -173,7 +197,7 @@ public class IdUserGroup { } map.put(key, value); } - LOG.info("Updated " + mapName + " map size:" + map.size()); + LOG.info("Updated " + mapName + " map size: " + map.size()); } catch (IOException e) { LOG.error("Can't update " + mapName + " map"); @@ -199,20 +223,115 @@ public class IdUserGroup { + " 'nobody' will be used for any user and group."); return; } + + StaticMapping staticMapping = new StaticMapping( + new HashMap(), new HashMap()); + if (staticMappingFile.exists()) { + LOG.info("Using '" + staticMappingFile + "' for static UID/GID mapping..."); + staticMapping = parseStaticMap(staticMappingFile); + } else { + LOG.info("Not doing static UID/GID mapping because '" + staticMappingFile + + "' does not exist."); + } if (OS.startsWith("Linux")) { - updateMapInternal(uMap, "user", LINUX_GET_ALL_USERS_CMD, ":"); - updateMapInternal(gMap, "group", LINUX_GET_ALL_GROUPS_CMD, ":"); + updateMapInternal(uMap, "user", LINUX_GET_ALL_USERS_CMD, ":", + staticMapping.uidMapping); + updateMapInternal(gMap, "group", LINUX_GET_ALL_GROUPS_CMD, ":", + staticMapping.gidMapping); } else { // Mac - updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+"); - updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+"); + updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+", + staticMapping.uidMapping); + updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+", + staticMapping.gidMapping); } uidNameMap = uMap; gidNameMap = gMap; lastUpdateTime = Time.monotonicNow(); } + + @SuppressWarnings("serial") + static final class PassThroughMap extends HashMap { + + public PassThroughMap() { + this(new HashMap()); + } + + public PassThroughMap(Map mapping) { + super(); + for (Map.Entry entry : mapping.entrySet()) { + super.put(entry.getKey(), entry.getValue()); + } + } + + @SuppressWarnings("unchecked") + @Override + public K get(Object key) { + if (super.containsKey(key)) { + return super.get(key); + } else { + return (K) key; + } + } + } + + @VisibleForTesting + static final class StaticMapping { + final Map uidMapping; + final Map gidMapping; + + public StaticMapping(Map uidMapping, + Map gidMapping) { + this.uidMapping = new PassThroughMap(uidMapping); + this.gidMapping = new PassThroughMap(gidMapping); + } + } + + static StaticMapping parseStaticMap(File staticMapFile) + throws IOException { + + Map uidMapping = new HashMap(); + Map gidMapping = new HashMap(); + + BufferedReader in = new BufferedReader(new InputStreamReader( + new FileInputStream(staticMapFile))); + + try { + String line = null; + while ((line = in.readLine()) != null) { + // Skip entirely empty and comment lines. + if (EMPTY_LINE.matcher(line).matches() || + COMMENT_LINE.matcher(line).matches()) { + continue; + } + + Matcher lineMatcher = MAPPING_LINE.matcher(line); + if (!lineMatcher.matches()) { + LOG.warn("Could not parse line '" + line + "'. Lines should be of " + + "the form '[uid|gid] [remote id] [local id]'. Blank lines and " + + "everything following a '#' on a line will be ignored."); + continue; + } + + // We know the line is fine to parse without error checking like this + // since it matched the regex above. + String firstComponent = lineMatcher.group(1); + int remoteId = Integer.parseInt(lineMatcher.group(2)); + int localId = Integer.parseInt(lineMatcher.group(3)); + if (firstComponent.equals("uid")) { + uidMapping.put(localId, remoteId); + } else { + gidMapping.put(localId, remoteId); + } + } + } finally { + in.close(); + } + + return new StaticMapping(uidMapping, gidMapping); + } synchronized public int getUid(String user) throws IOException { checkAndUpdateMaps(); diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java index 77477ff1693..bca53d0fa36 100644 --- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/nfs/nfs3/TestIdUserGroup.java @@ -19,15 +19,97 @@ package org.apache.hadoop.nfs.nfs3; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.nfs.nfs3.IdUserGroup.PassThroughMap; +import org.apache.hadoop.nfs.nfs3.IdUserGroup.StaticMapping; import org.junit.Test; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; public class TestIdUserGroup { + + private static final Map EMPTY_PASS_THROUGH_MAP = + new PassThroughMap(); + + @Test + public void testStaticMapParsing() throws IOException { + File tempStaticMapFile = File.createTempFile("nfs-", ".map"); + final String staticMapFileContents = + "uid 10 100\n" + + "gid 10 200\n" + + "uid 11 201 # comment at the end of a line\n" + + "uid 12 301\n" + + "# Comment at the beginning of a line\n" + + " # Comment that starts late in the line\n" + + "uid 10000 10001# line without whitespace before comment\n" + + "uid 13 302\n" + + "gid\t11\t201\n" + // Tabs instead of spaces. + "\n" + // Entirely empty line. + "gid 12 202"; + OutputStream out = new FileOutputStream(tempStaticMapFile); + out.write(staticMapFileContents.getBytes()); + out.close(); + StaticMapping parsedMap = IdUserGroup.parseStaticMap(tempStaticMapFile); + + assertEquals(10, (int)parsedMap.uidMapping.get(100)); + assertEquals(11, (int)parsedMap.uidMapping.get(201)); + assertEquals(12, (int)parsedMap.uidMapping.get(301)); + assertEquals(13, (int)parsedMap.uidMapping.get(302)); + assertEquals(10, (int)parsedMap.gidMapping.get(200)); + assertEquals(11, (int)parsedMap.gidMapping.get(201)); + assertEquals(12, (int)parsedMap.gidMapping.get(202)); + assertEquals(10000, (int)parsedMap.uidMapping.get(10001)); + // Ensure pass-through of unmapped IDs works. + assertEquals(1000, (int)parsedMap.uidMapping.get(1000)); + } + + @Test + public void testStaticMapping() throws IOException { + Map uidStaticMap = new PassThroughMap(); + Map gidStaticMap = new PassThroughMap(); + + uidStaticMap.put(11501, 10); + gidStaticMap.put(497, 200); + + // Maps for id to name map + BiMap uMap = HashBiMap.create(); + BiMap gMap = HashBiMap.create(); + + String GET_ALL_USERS_CMD = + "echo \"atm:x:1000:1000:Aaron T. Myers,,,:/home/atm:/bin/bash\n" + + "hdfs:x:11501:10787:Grid Distributed File System:/home/hdfs:/bin/bash\"" + + " | cut -d: -f1,3"; + + String GET_ALL_GROUPS_CMD = "echo \"hdfs:*:11501:hrt_hdfs\n" + + "mapred:x:497\n" + + "mapred2:x:498\"" + + " | cut -d: -f1,3"; + + IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":", + uidStaticMap); + IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":", + gidStaticMap); + + assertEquals("hdfs", uMap.get(10)); + assertEquals(10, (int)uMap.inverse().get("hdfs")); + assertEquals("atm", uMap.get(1000)); + assertEquals(1000, (int)uMap.inverse().get("atm")); + + assertEquals("hdfs", gMap.get(11501)); + assertEquals(11501, (int)gMap.inverse().get("hdfs")); + assertEquals("mapred", gMap.get(200)); + assertEquals(200, (int)gMap.inverse().get("mapred")); + assertEquals("mapred2", gMap.get(498)); + assertEquals(498, (int)gMap.inverse().get("mapred2")); + } @Test public void testDuplicates() throws IOException { @@ -51,15 +133,17 @@ public class TestIdUserGroup { BiMap uMap = HashBiMap.create(); BiMap gMap = HashBiMap.create(); - IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":"); - assertTrue(uMap.size() == 5); + IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":", + EMPTY_PASS_THROUGH_MAP); + assertEquals(5, uMap.size()); assertEquals("root", uMap.get(0)); assertEquals("hdfs", uMap.get(11501)); assertEquals("hdfs2",uMap.get(11502)); assertEquals("bin", uMap.get(2)); assertEquals("daemon", uMap.get(1)); - IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":"); + IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":", + EMPTY_PASS_THROUGH_MAP); assertTrue(gMap.size() == 3); assertEquals("hdfs",gMap.get(11501)); assertEquals("mapred", gMap.get(497)); @@ -90,7 +174,8 @@ public class TestIdUserGroup { BiMap uMap = HashBiMap.create(); BiMap gMap = HashBiMap.create(); - IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":"); + IdUserGroup.updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":", + EMPTY_PASS_THROUGH_MAP); assertTrue(uMap.size() == 7); assertEquals("nfsnobody", uMap.get(-2)); assertEquals("nfsnobody1", uMap.get(-1)); @@ -100,7 +185,8 @@ public class TestIdUserGroup { assertEquals("hdfs",uMap.get(11501)); assertEquals("daemon", uMap.get(2)); - IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":"); + IdUserGroup.updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":", + EMPTY_PASS_THROUGH_MAP); assertTrue(gMap.size() == 7); assertEquals("hdfs",gMap.get(11501)); assertEquals("rpcuser", gMap.get(29)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6987595891e..f0995e06582 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -440,6 +440,9 @@ Release 2.5.0 - UNRELEASED HDFS-6396. Remove support for ACL feature from INodeSymlink. (Charles Lamb via wang) + HDFS-6435. Add support for specifying a static uid/gid mapping for the NFS + gateway. (atm via wang) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm index 8f5bef64aef..7f8e8211b10 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm @@ -338,8 +338,21 @@ HDFS NFS Gateway The system administrator must ensure that the user on NFS client host has the same name and UID as that on the NFS gateway host. This is usually not a problem if the same user management system (e.g., LDAP/NIS) is used to create and deploy users on - HDFS nodes and NFS client node. In case the user account is created manually in different hosts, one might need to + HDFS nodes and NFS client node. In case the user account is created manually on different hosts, one might need to modify UID (e.g., do "usermod -u 123 myusername") on either NFS client or NFS gateway host in order to make it the same on both sides. More technical details of RPC AUTH_UNIX can be found in {{{http://tools.ietf.org/html/rfc1057}RPC specification}}. + Optionally, the system administrator can configure a custom static mapping + file in the event one wishes to access the HDFS NFS Gateway from a system with + a completely disparate set of UIDs/GIDs. By default this file is located at + "/etc/nfs.map", but a custom location can be configured by setting the + "dfs.nfs.static.mapping.file" property to the path of the static mapping file. + The format of the static mapping file is similar to what is described in the + exports(5) manual page, but roughly it is: + +------------------------- +# Mapping for clients accessing the NFS gateway +uid 10 100 # Map the remote UID 10 the local UID 100 +gid 11 101 # Map the remote GID 11 to the local GID 101 +------------------------- From b867b695565c588e8f86c867cba76397cab62848 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 23 May 2014 17:09:47 +0000 Subject: [PATCH 04/34] YARN-2049. Added delegation-token support for the Timeline Server. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597130 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../TimelineDelegationTokenResponse.java | 63 +++++ .../hadoop/yarn/conf/YarnConfiguration.java | 8 - .../yarn/client/api/TimelineClient.java | 2 - .../api/impl/TimelineAuthenticator.java | 252 ++++++++++++++++++ .../client/ClientTimelineSecurityInfo.java | 83 ++++++ .../client/TimelineAuthenticationConsts.java | 44 +++ .../TimelineDelegationTokenIdentifier.java | 63 +++++ .../TimelineDelegationTokenOperation.java | 55 ++++ .../TimelineDelegationTokenSelector.java | 62 +++++ .../yarn/util/timeline/TimelineUtils.java | 8 +- .../webapp/YarnJacksonJaxbJsonProvider.java | 7 +- .../org.apache.hadoop.security.SecurityInfo | 1 + ...ache.hadoop.security.token.TokenIdentifier | 1 + ....apache.hadoop.security.token.TokenRenewer | 1 + .../ApplicationHistoryServer.java | 54 +++- .../TimelineAuthenticationFilter.java | 48 ++++ ...melineAuthenticationFilterInitializer.java | 127 +++++++++ .../TimelineClientAuthenticationService.java | 236 ++++++++++++++++ ...neDelegationTokenSecretManagerService.java | 189 +++++++++++++ .../webapp/AHSWebApp.java | 55 +++- .../TestApplicationHistoryClientService.java | 2 + .../TestApplicationHistoryServer.java | 4 +- .../hadoop/yarn/server/MiniYARNCluster.java | 2 + 24 files changed, 1335 insertions(+), 35 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineAuthenticationConsts.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenOperation.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index fe4d3ac05b0..93764e67e84 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -90,6 +90,9 @@ Release 2.5.0 - UNRELEASED YARN-2017. Merged some of the common scheduler code. (Jian He via vinodkv) + YARN-2049. Added delegation-token support for the Timeline Server. (Zhijie + Shen via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java new file mode 100644 index 00000000000..90d13a2b0c4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.timeline; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The response of delegation token related request + */ +@XmlRootElement(name = "delegationtoken") +@XmlAccessorType(XmlAccessType.NONE) +@Public +@Unstable +public class TimelineDelegationTokenResponse { + + private String type; + private Object content; + + public TimelineDelegationTokenResponse() { + + } + + @XmlElement(name = "type") + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + @XmlElement(name = "content") + public Object getContent() { + return content; + } + + public void setContent(Object content) { + this.content = content; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e25a941f970..48763b3120e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1136,14 +1136,6 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; - /**The kerberos principal to be used for spnego filter for timeline service.*/ - public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY = - TIMELINE_SERVICE_PREFIX + "webapp.spnego-principal"; - - /**The kerberos keytab to be used for spnego filter for timeline service.*/ - public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = - TIMELINE_SERVICE_PREFIX + "webapp.spnego-keytab-file"; - /** Timeline service store class */ public static final String TIMELINE_SERVICE_STORE = TIMELINE_SERVICE_PREFIX + "store-class"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java index fda0a4f18e6..a2ed3e70a51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java @@ -32,8 +32,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException; /** * A client library that can be used to post some information in terms of a * number of conceptual entities. - * - * @See Entity */ @Public @Unstable diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java new file mode 100644 index 00000000000..f4f1507a4f2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java @@ -0,0 +1,252 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api.impl; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.Authenticator; +import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenOperation; +import org.apache.hadoop.yarn.security.client.TimelineAuthenticationConsts; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * A KerberosAuthenticator subclass that fallback to + * {@link TimelineAuthenticationConsts}. + */ +@Private +@Unstable +public class TimelineAuthenticator extends KerberosAuthenticator { + + private static ObjectMapper mapper; + + static { + mapper = new ObjectMapper(); + YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); + } + + /** + * Returns the fallback authenticator if the server does not use Kerberos + * SPNEGO HTTP authentication. + * + * @return a {@link TimelineAuthenticationConsts} instance. + */ + @Override + protected Authenticator getFallBackAuthenticator() { + return new TimelineAuthenticator(); + } + + public static void injectDelegationToken(Map params, + Token dtToken) + throws IOException { + if (dtToken != null) { + params.put(TimelineAuthenticationConsts.DELEGATION_PARAM, + dtToken.encodeToUrlString()); + } + } + + private boolean hasDelegationToken(URL url) { + return url.getQuery().contains( + TimelineAuthenticationConsts.DELEGATION_PARAM + "="); + } + + @Override + public void authenticate(URL url, AuthenticatedURL.Token token) + throws IOException, AuthenticationException { + if (!hasDelegationToken(url)) { + super.authenticate(url, token); + } + } + + public static Token getDelegationToken( + URL url, AuthenticatedURL.Token token, String renewer) throws IOException { + TimelineDelegationTokenOperation op = + TimelineDelegationTokenOperation.GETDELEGATIONTOKEN; + Map params = new HashMap(); + params.put(TimelineAuthenticationConsts.OP_PARAM, op.toString()); + params.put(TimelineAuthenticationConsts.RENEWER_PARAM, renewer); + url = appendParams(url, params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new TimelineAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod(op.getHttpMethod()); + TimelineDelegationTokenResponse dtRes = validateAndParseResponse(conn); + if (!dtRes.getType().equals( + TimelineAuthenticationConsts.DELEGATION_TOKEN_URL)) { + throw new IOException("The response content is not expected: " + + dtRes.getContent()); + } + String tokenStr = dtRes.getContent().toString(); + Token dToken = + new Token(); + dToken.decodeFromUrlString(tokenStr); + return dToken; + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + + public static long renewDelegationToken(URL url, + AuthenticatedURL.Token token, + Token dToken) throws IOException { + Map params = new HashMap(); + params.put(TimelineAuthenticationConsts.OP_PARAM, + TimelineDelegationTokenOperation.RENEWDELEGATIONTOKEN.toString()); + params.put(TimelineAuthenticationConsts.TOKEN_PARAM, + dToken.encodeToUrlString()); + url = appendParams(url, params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new TimelineAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod( + TimelineDelegationTokenOperation.RENEWDELEGATIONTOKEN.getHttpMethod()); + TimelineDelegationTokenResponse dtRes = validateAndParseResponse(conn); + if (!dtRes.getType().equals( + TimelineAuthenticationConsts.DELEGATION_TOKEN_EXPIRATION_TIME)) { + throw new IOException("The response content is not expected: " + + dtRes.getContent()); + } + return Long.valueOf(dtRes.getContent().toString()); + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + + public static void cancelDelegationToken(URL url, + AuthenticatedURL.Token token, + Token dToken) throws IOException { + Map params = new HashMap(); + params.put(TimelineAuthenticationConsts.OP_PARAM, + TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN.toString()); + params.put(TimelineAuthenticationConsts.TOKEN_PARAM, + dToken.encodeToUrlString()); + url = appendParams(url, params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new TimelineAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod(TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN + .getHttpMethod()); + validateAndParseResponse(conn); + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + + /** + * Convenience method that appends parameters an HTTP URL. + * + * @param url + * the url. + * @param params + * the query string parameters. + * + * @return a URL + * + * @throws IOException + * thrown if an IO error occurs. + */ + public static URL appendParams(URL url, Map params) + throws IOException { + StringBuilder sb = new StringBuilder(); + sb.append(url); + String separator = url.toString().contains("?") ? "&" : "?"; + for (Map.Entry entry : params.entrySet()) { + sb.append(separator).append(entry.getKey()).append("="). + append(URLEncoder.encode(entry.getValue(), "UTF8")); + separator = "&"; + } + return new URL(sb.toString()); + } + + /** + * Validates the response of an HttpURLConnection. If the current + * status code is not 200, it will throw an exception with a detail message + * using Server side error messages if available. Otherwise, + * {@link TimelineDelegationTokenResponse} will be parsed and returned. + * + * @param conn + * the HttpURLConnection. + * @return + * @throws IOException + * thrown if the current status code is not 200 or the JSON response + * cannot be parsed correctly + */ + private static TimelineDelegationTokenResponse validateAndParseResponse( + HttpURLConnection conn) throws IOException { + int status = conn.getResponseCode(); + JsonNode json = mapper.readTree(conn.getInputStream()); + if (status == HttpURLConnection.HTTP_OK) { + return mapper.readValue(json, TimelineDelegationTokenResponse.class); + } else { + // If the status code is not 200, some thing wrong should happen at the + // server side, the JSON content is going to contain exception details. + // We can use the JSON content to reconstruct the exception object. + try { + String message = + json.get(TimelineAuthenticationConsts.ERROR_MESSAGE_JSON) + .getTextValue(); + String exception = + json.get(TimelineAuthenticationConsts.ERROR_EXCEPTION_JSON) + .getTextValue(); + String className = + json.get(TimelineAuthenticationConsts.ERROR_CLASSNAME_JSON) + .getTextValue(); + + try { + ClassLoader cl = TimelineAuthenticator.class.getClassLoader(); + Class klass = cl.loadClass(className); + Constructor constr = klass.getConstructor(String.class); + throw (IOException) constr.newInstance(message); + } catch (IOException ex) { + throw ex; + } catch (Exception ex) { + throw new IOException(MessageFormat.format("{0} - {1}", exception, + message)); + } + } catch (IOException ex) { + if (ex.getCause() instanceof IOException) { + throw (IOException) ex.getCause(); + } + throw new IOException( + MessageFormat.format("HTTP status [{0}], {1}", + status, conn.getResponseMessage())); + } + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java new file mode 100644 index 00000000000..d2b135851a5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import java.lang.annotation.Annotation; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.KerberosInfo; +import org.apache.hadoop.security.SecurityInfo; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenInfo; +import org.apache.hadoop.security.token.TokenSelector; +import org.apache.hadoop.yarn.api.ApplicationHistoryProtocolPB; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + + +@Public +@Unstable +public class ClientTimelineSecurityInfo extends SecurityInfo { + @Override + public KerberosInfo getKerberosInfo(Class protocol, Configuration conf) { + if (!protocol + .equals(ApplicationHistoryProtocolPB.class)) { + return null; + } + return new KerberosInfo() { + + @Override + public Class annotationType() { + return null; + } + + @Override + public String serverPrincipal() { + return YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL; + } + + @Override + public String clientPrincipal() { + return null; + } + }; + } + + @Override + public TokenInfo getTokenInfo(Class protocol, Configuration conf) { + if (!protocol + .equals(ApplicationHistoryProtocolPB.class)) { + return null; + } + return new TokenInfo() { + + @Override + public Class annotationType() { + return null; + } + + @Override + public Class> + value() { + return TimelineDelegationTokenSelector.class; + } + }; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineAuthenticationConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineAuthenticationConsts.java new file mode 100644 index 00000000000..e89e1a66aa1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineAuthenticationConsts.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The constants that are going to be used by the timeline Kerberos + delegation + * token authentication. + */ + +@Private +@Unstable +public class TimelineAuthenticationConsts { + + public static final String ERROR_EXCEPTION_JSON = "exception"; + public static final String ERROR_CLASSNAME_JSON = "javaClassName"; + public static final String ERROR_MESSAGE_JSON = "message"; + + public static final String OP_PARAM = "op"; + public static final String DELEGATION_PARAM = "delegation"; + public static final String TOKEN_PARAM = "token"; + public static final String RENEWER_PARAM = "renewer"; + public static final String DELEGATION_TOKEN_URL = "url"; + public static final String DELEGATION_TOKEN_EXPIRATION_TIME = + "expirationTime"; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java new file mode 100644 index 00000000000..82e0d69c573 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; + +@Public +@Unstable +public class TimelineDelegationTokenIdentifier extends AbstractDelegationTokenIdentifier { + + public static final Text KIND_NAME = new Text("TIMELINE_DELEGATION_TOKEN"); + + public TimelineDelegationTokenIdentifier() { + + } + + /** + * Create a new timeline delegation token identifier + * + * @param owner the effective username of the token owner + * @param renewer the username of the renewer + * @param realUser the real username of the token owner + */ + public TimelineDelegationTokenIdentifier(Text owner, Text renewer, + Text realUser) { + super(owner, renewer, realUser); + } + + @Override + public Text getKind() { + return KIND_NAME; + } + + @InterfaceAudience.Private + public static class Renewer extends Token.TrivialRenewer { + @Override + protected Text getKind() { + return KIND_NAME; + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenOperation.java new file mode 100644 index 00000000000..33ab17829f0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenOperation.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPut; + +/** + * DelegationToken operations. + */ +@Unstable +@Private +public enum TimelineDelegationTokenOperation { + // TODO: need think about which ops can be done without kerberos + // credentials, for safety, we enforces all need kerberos credentials now. + GETDELEGATIONTOKEN(HttpGet.METHOD_NAME, true), + RENEWDELEGATIONTOKEN(HttpPut.METHOD_NAME, true), + CANCELDELEGATIONTOKEN(HttpPut.METHOD_NAME, true); + + private String httpMethod; + private boolean requiresKerberosCredentials; + + private TimelineDelegationTokenOperation(String httpMethod, + boolean requiresKerberosCredentials) { + this.httpMethod = httpMethod; + this.requiresKerberosCredentials = requiresKerberosCredentials; + } + + public String getHttpMethod() { + return httpMethod; + } + + public boolean requiresKerberosCredentials() { + return requiresKerberosCredentials; + } + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java new file mode 100644 index 00000000000..df1e84d5006 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import java.util.Collection; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenSelector; + +@Public +@Unstable +public class TimelineDelegationTokenSelector + implements TokenSelector { + + private static final Log LOG = LogFactory + .getLog(TimelineDelegationTokenSelector.class); + + @SuppressWarnings("unchecked") + public Token selectToken(Text service, + Collection> tokens) { + if (service == null) { + return null; + } + if (LOG.isDebugEnabled()) { + LOG.debug("Looking for a token with service " + service.toString()); + } + for (Token token : tokens) { + if (LOG.isDebugEnabled()) { + LOG.debug("Token kind is " + token.getKind().toString() + + " and the token's service name is " + token.getService()); + } + if (TimelineDelegationTokenIdentifier.KIND_NAME.equals(token.getKind()) + && service.equals(token.getService())) { + return (Token) token; + } + } + return null; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java index 324c6f663d5..a62ed4869da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java @@ -22,12 +22,10 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.codehaus.jackson.JsonGenerationException; -import org.codehaus.jackson.map.AnnotationIntrospector; import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; -import org.codehaus.jackson.xc.JaxbAnnotationIntrospector; /** * The helper class for the timeline module. @@ -41,9 +39,7 @@ public class TimelineUtils { static { mapper = new ObjectMapper(); - AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); - mapper.setAnnotationIntrospector(introspector); - mapper.setSerializationInclusion(Inclusion.NON_NULL); + YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java index ac0ea7f9832..3cc1aec5c6f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java @@ -49,9 +49,14 @@ public class YarnJacksonJaxbJsonProvider extends JacksonJaxbJsonProvider { @Override public ObjectMapper locateMapper(Class type, MediaType mediaType) { ObjectMapper mapper = super.locateMapper(type, mediaType); + configObjectMapper(mapper); + return mapper; + } + + public static void configObjectMapper(ObjectMapper mapper) { AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); mapper.setAnnotationIntrospector(introspector); mapper.setSerializationInclusion(Inclusion.NON_NULL); - return mapper; } + } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo index babc2fbf8e4..d90a26703ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo @@ -12,6 +12,7 @@ # limitations under the License. # org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo +org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo org.apache.hadoop.yarn.security.SchedulerSecurityInfo org.apache.hadoop.yarn.security.admin.AdminSecurityInfo diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier index d860461467e..a4ad54813d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -15,4 +15,5 @@ org.apache.hadoop.yarn.security.ContainerTokenIdentifier org.apache.hadoop.yarn.security.AMRMTokenIdentifier org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier +org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier org.apache.hadoop.yarn.security.NMTokenIdentifier diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer index dd0b2c48def..9fcfa19465f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -14,3 +14,4 @@ org.apache.hadoop.yarn.security.AMRMTokenIdentifier$Renewer org.apache.hadoop.yarn.security.ContainerTokenIdentifier$Renewer org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier$Renewer +org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier$Renewer diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 9f98834f7ce..a2f5a2489e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.Service; import org.apache.hadoop.util.ExitUtil; @@ -39,6 +40,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineAuthenticationFilterInitializer; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; @@ -56,10 +59,11 @@ public class ApplicationHistoryServer extends CompositeService { private static final Log LOG = LogFactory .getLog(ApplicationHistoryServer.class); - ApplicationHistoryClientService ahsClientService; - ApplicationHistoryManager historyManager; - TimelineStore timelineStore; - private WebApp webApp; + protected ApplicationHistoryClientService ahsClientService; + protected ApplicationHistoryManager historyManager; + protected TimelineStore timelineStore; + protected TimelineDelegationTokenSecretManagerService secretManagerService; + protected WebApp webApp; public ApplicationHistoryServer() { super(ApplicationHistoryServer.class.getName()); @@ -73,6 +77,8 @@ public class ApplicationHistoryServer extends CompositeService { addService((Service) historyManager); timelineStore = createTimelineStore(conf); addIfService(timelineStore); + secretManagerService = createTimelineDelegationTokenSecretManagerService(conf); + addService(secretManagerService); DefaultMetricsSystem.initialize("ApplicationHistoryServer"); JvmMetrics.initSingleton("ApplicationHistoryServer", null); @@ -158,21 +164,43 @@ public class ApplicationHistoryServer extends CompositeService { TimelineStore.class), conf); } + protected TimelineDelegationTokenSecretManagerService + createTimelineDelegationTokenSecretManagerService(Configuration conf) { + return new TimelineDelegationTokenSecretManagerService(); + } + protected void startWebApp() { - String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig()); + Configuration conf = getConfig(); + // Play trick to make the customized filter will only be loaded by the + // timeline server when security is enabled and Kerberos authentication + // is used. + if (UserGroupInformation.isSecurityEnabled() + && conf + .get(TimelineAuthenticationFilterInitializer.PREFIX + "type", "") + .equals("kerberos")) { + String initializers = conf.get("hadoop.http.filter.initializers"); + initializers = + initializers == null || initializers.length() == 0 ? "" : "," + + initializers; + if (!initializers.contains( + TimelineAuthenticationFilterInitializer.class.getName())) { + conf.set("hadoop.http.filter.initializers", + TimelineAuthenticationFilterInitializer.class.getName() + + initializers); + } + } + String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(conf); LOG.info("Instantiating AHSWebApp at " + bindAddress); try { + AHSWebApp ahsWebApp = AHSWebApp.getInstance(); + ahsWebApp.setApplicationHistoryManager(historyManager); + ahsWebApp.setTimelineStore(timelineStore); + ahsWebApp.setTimelineDelegationTokenSecretManagerService(secretManagerService); webApp = WebApps .$for("applicationhistory", ApplicationHistoryClientService.class, - ahsClientService, "ws") - .with(getConfig()) - .withHttpSpnegoPrincipalKey( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY) - .withHttpSpnegoKeytabKey( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) - .at(bindAddress) - .start(new AHSWebApp(historyManager, timelineStore)); + ahsClientService, "ws") + .with(conf).at(bindAddress).start(ahsWebApp); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java new file mode 100644 index 00000000000..53ef1ed3e84 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import java.util.Properties; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; + +@Private +@Unstable +public class TimelineAuthenticationFilter extends AuthenticationFilter { + + @Override + protected Properties getConfiguration(String configPrefix, + FilterConfig filterConfig) throws ServletException { + // In yarn-site.xml, we can simply set type to "kerberos". However, we need + // to replace the name here to use the customized Kerberos + DT service + // instead of the standard Kerberos handler. + Properties properties = super.getConfiguration(configPrefix, filterConfig); + if (properties.getProperty(AUTH_TYPE).equals("kerberos")) { + properties.setProperty( + AUTH_TYPE, TimelineClientAuthenticationService.class.getName()); + } + return properties; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java new file mode 100644 index 00000000000..e3c303233bb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import java.io.FileReader; +import java.io.IOException; +import java.io.Reader; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.FilterContainer; +import org.apache.hadoop.http.FilterInitializer; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.security.SecurityUtil; + +/** + *

+ * Initializes {@link TimelineAuthenticationFilter} which provides support for + * Kerberos HTTP SPNEGO authentication. + *

+ *

+ * It enables Kerberos HTTP SPNEGO plus delegation token authentication for the + * timeline server. + *

+ * Refer to the core-default.xml file, after the comment 'HTTP + * Authentication' for details on the configuration options. All related + * configuration properties have 'hadoop.http.authentication.' as prefix. + */ +public class TimelineAuthenticationFilterInitializer extends FilterInitializer { + + /** + * The configuration prefix of timeline Kerberos + DT authentication + */ + public static final String PREFIX = "yarn.timeline-service.http.authentication."; + + private static final String SIGNATURE_SECRET_FILE = + TimelineAuthenticationFilter.SIGNATURE_SECRET + ".file"; + + /** + *

+ * Initializes {@link TimelineAuthenticationFilter} + *

+ *

+ * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN + * configuration properties prefixed with + * "yarn.timeline-service.authentication." + *

+ * + * @param container + * The filter container + * @param conf + * Configuration for run-time parameters + */ + @Override + public void initFilter(FilterContainer container, Configuration conf) { + Map filterConfig = new HashMap(); + + // setting the cookie path to root '/' so it is used for all resources. + filterConfig.put(TimelineAuthenticationFilter.COOKIE_PATH, "/"); + + for (Map.Entry entry : conf) { + String name = entry.getKey(); + if (name.startsWith(PREFIX)) { + String value = conf.get(name); + name = name.substring(PREFIX.length()); + filterConfig.put(name, value); + } + } + + String signatureSecretFile = filterConfig.get(SIGNATURE_SECRET_FILE); + if (signatureSecretFile != null) { + try { + StringBuilder secret = new StringBuilder(); + Reader reader = new FileReader(signatureSecretFile); + int c = reader.read(); + while (c > -1) { + secret.append((char) c); + c = reader.read(); + } + reader.close(); + filterConfig + .put(TimelineAuthenticationFilter.SIGNATURE_SECRET, + secret.toString()); + } catch (IOException ex) { + throw new RuntimeException( + "Could not read HTTP signature secret file: " + + signatureSecretFile); + } + } + + // Resolve _HOST into bind address + String bindAddress = conf.get(HttpServer2.BIND_ADDRESS); + String principal = + filterConfig.get(TimelineClientAuthenticationService.PRINCIPAL); + if (principal != null) { + try { + principal = SecurityUtil.getServerPrincipal(principal, bindAddress); + } catch (IOException ex) { + throw new RuntimeException( + "Could not resolve Kerberos principal name: " + ex.toString(), ex); + } + filterConfig.put(TimelineClientAuthenticationService.PRINCIPAL, + principal); + } + + container.addGlobalFilter("Timeline Authentication Filter", + TimelineAuthenticationFilter.class.getName(), + filterConfig); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java new file mode 100644 index 00000000000..f11633d9697 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java @@ -0,0 +1,236 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import java.io.IOException; +import java.io.Writer; +import java.text.MessageFormat; +import java.util.HashSet; +import java.util.Set; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.MediaType; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenOperation; +import org.apache.hadoop.yarn.security.client.TimelineAuthenticationConsts; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * Server side AuthenticationHandler that authenticates requests + * using the incoming delegation token as a 'delegation' query string parameter. + *

+ * If not delegation token is present in the request it delegates to the + * {@link KerberosAuthenticationHandler} + */ +@Private +@Unstable +public class TimelineClientAuthenticationService + extends KerberosAuthenticationHandler { + + public static final String TYPE = "kerberos-dt"; + private static final Set DELEGATION_TOKEN_OPS = new HashSet(); + private static final String OP_PARAM = "op"; + private static final String ENTER = System.getProperty("line.separator"); + + private ObjectMapper mapper; + + static { + DELEGATION_TOKEN_OPS.add( + TimelineDelegationTokenOperation.GETDELEGATIONTOKEN.toString()); + DELEGATION_TOKEN_OPS.add( + TimelineDelegationTokenOperation.RENEWDELEGATIONTOKEN.toString()); + DELEGATION_TOKEN_OPS.add( + TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN.toString()); + } + + public TimelineClientAuthenticationService() { + super(); + mapper = new ObjectMapper(); + YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); + } + + /** + * Returns authentication type of the handler. + * + * @return delegationtoken-kerberos + */ + @Override + public String getType() { + return TYPE; + } + + @Override + public boolean managementOperation(AuthenticationToken token, + HttpServletRequest request, HttpServletResponse response) + throws IOException, AuthenticationException { + boolean requestContinues = true; + String op = request.getParameter(OP_PARAM); + op = (op != null) ? op.toUpperCase() : null; + if (DELEGATION_TOKEN_OPS.contains(op) && + !request.getMethod().equals("OPTIONS")) { + TimelineDelegationTokenOperation dtOp = + TimelineDelegationTokenOperation.valueOf(op); + if (dtOp.getHttpMethod().equals(request.getMethod())) { + if (dtOp.requiresKerberosCredentials() && token == null) { + response.sendError(HttpServletResponse.SC_UNAUTHORIZED, + MessageFormat.format( + "Operation [{0}] requires SPNEGO authentication established", + dtOp)); + requestContinues = false; + } else { + TimelineDelegationTokenSecretManagerService secretManager = + AHSWebApp.getInstance() + .getTimelineDelegationTokenSecretManagerService(); + try { + TimelineDelegationTokenResponse res = null; + switch (dtOp) { + case GETDELEGATIONTOKEN: + UserGroupInformation ownerUGI = + UserGroupInformation.createRemoteUser(token.getUserName()); + String renewerParam = + request + .getParameter(TimelineAuthenticationConsts.RENEWER_PARAM); + if (renewerParam == null) { + renewerParam = token.getUserName(); + } + Token dToken = + secretManager.createToken(ownerUGI, renewerParam); + res = new TimelineDelegationTokenResponse(); + res.setType(TimelineAuthenticationConsts.DELEGATION_TOKEN_URL); + res.setContent(dToken.encodeToUrlString()); + break; + case RENEWDELEGATIONTOKEN: + case CANCELDELEGATIONTOKEN: + String tokenParam = + request + .getParameter(TimelineAuthenticationConsts.TOKEN_PARAM); + if (tokenParam == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, + MessageFormat + .format( + "Operation [{0}] requires the parameter [{1}]", + dtOp, + TimelineAuthenticationConsts.TOKEN_PARAM)); + requestContinues = false; + } else { + if (dtOp == TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN) { + Token dt = + new Token(); + dt.decodeFromUrlString(tokenParam); + secretManager.cancelToken(dt, token.getUserName()); + } else { + Token dt = + new Token(); + dt.decodeFromUrlString(tokenParam); + long expirationTime = + secretManager.renewToken(dt, token.getUserName()); + res = new TimelineDelegationTokenResponse(); + res.setType(TimelineAuthenticationConsts.DELEGATION_TOKEN_EXPIRATION_TIME); + res.setContent(expirationTime); + } + } + break; + } + if (requestContinues) { + response.setStatus(HttpServletResponse.SC_OK); + if (res != null) { + response.setContentType(MediaType.APPLICATION_JSON); + Writer writer = response.getWriter(); + mapper.writeValue(writer, res); + writer.write(ENTER); + writer.flush(); + + } + requestContinues = false; + } + } catch (IOException e) { + throw new AuthenticationException(e.toString(), e); + } + } + } else { + response + .sendError( + HttpServletResponse.SC_BAD_REQUEST, + MessageFormat + .format( + "Wrong HTTP method [{0}] for operation [{1}], it should be [{2}]", + request.getMethod(), dtOp, dtOp.getHttpMethod())); + requestContinues = false; + } + } + return requestContinues; + } + + /** + * Authenticates a request looking for the delegation + * query-string parameter and verifying it is a valid token. If there is not + * delegation query-string parameter, it delegates the + * authentication to the {@link KerberosAuthenticationHandler} unless it is + * disabled. + * + * @param request + * the HTTP client request. + * @param response + * the HTTP client response. + * + * @return the authentication token for the authenticated request. + * @throws IOException + * thrown if an IO error occurred. + * @throws AuthenticationException + * thrown if the authentication failed. + */ + @Override + public AuthenticationToken authenticate(HttpServletRequest request, + HttpServletResponse response) + throws IOException, AuthenticationException { + AuthenticationToken token; + String delegationParam = + request + .getParameter(TimelineAuthenticationConsts.DELEGATION_PARAM); + if (delegationParam != null) { + Token dt = + new Token(); + dt.decodeFromUrlString(delegationParam); + TimelineDelegationTokenSecretManagerService secretManager = + AHSWebApp.getInstance() + .getTimelineDelegationTokenSecretManagerService(); + UserGroupInformation ugi = secretManager.verifyToken(dt); + final String shortName = ugi.getShortUserName(); + // creating a ephemeral token + token = new AuthenticationToken(shortName, ugi.getUserName(), getType()); + token.setExpires(0); + } else { + token = super.authenticate(request, response); + } + return token; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java new file mode 100644 index 00000000000..fee9eb41cd8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; + +/** + * The service wrapper of {@link TimelineDelegationTokenSecretManager} + */ +@Private +@Unstable +public class TimelineDelegationTokenSecretManagerService extends AbstractService { + + private TimelineDelegationTokenSecretManager secretManager = null; + private InetSocketAddress serviceAddr = null; + + public TimelineDelegationTokenSecretManagerService() { + super(TimelineDelegationTokenSecretManagerService.class.getName()); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + long secretKeyInterval = + conf.getLong(YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_KEY, + YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT); + long tokenMaxLifetime = + conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, + YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); + long tokenRenewInterval = + conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, + YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); + secretManager = new TimelineDelegationTokenSecretManager(secretKeyInterval, + tokenMaxLifetime, tokenRenewInterval, + 3600000); + secretManager.startThreads(); + + if (YarnConfiguration.useHttps(getConfig())) { + serviceAddr = getConfig().getSocketAddr( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT); + } else { + serviceAddr = getConfig().getSocketAddr( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT); + } + super.init(conf); + } + + @Override + protected void serviceStop() throws Exception { + secretManager.stopThreads(); + super.stop(); + } + + /** + * Creates a delegation token. + * + * @param ugi UGI creating the token. + * @param renewer token renewer. + * @return new delegation token. + * @throws IOException thrown if the token could not be created. + */ + public Token createToken( + UserGroupInformation ugi, String renewer) throws IOException { + renewer = (renewer == null) ? ugi.getShortUserName() : renewer; + String user = ugi.getUserName(); + Text owner = new Text(user); + Text realUser = null; + if (ugi.getRealUser() != null) { + realUser = new Text(ugi.getRealUser().getUserName()); + } + TimelineDelegationTokenIdentifier tokenIdentifier = + new TimelineDelegationTokenIdentifier(owner, new Text(renewer), realUser); + Token token = + new Token(tokenIdentifier, secretManager); + SecurityUtil.setTokenService(token, serviceAddr); + return token; + } + + /** + * Renews a delegation token. + * + * @param token delegation token to renew. + * @param renewer token renewer. + * @throws IOException thrown if the token could not be renewed. + */ + public long renewToken(Token token, + String renewer) throws IOException { + return secretManager.renewToken(token, renewer); + } + + /** + * Cancels a delegation token. + * + * @param token delegation token to cancel. + * @param canceler token canceler. + * @throws IOException thrown if the token could not be canceled. + */ + public void cancelToken(Token token, + String canceler) throws IOException { + secretManager.cancelToken(token, canceler); + } + + /** + * Verifies a delegation token. + * + * @param token delegation token to verify. + * @return the UGI for the token. + * @throws IOException thrown if the token could not be verified. + */ + public UserGroupInformation verifyToken(Token token) + throws IOException { + ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); + DataInputStream dis = new DataInputStream(buf); + TimelineDelegationTokenIdentifier id = new TimelineDelegationTokenIdentifier(); + try { + id.readFields(dis); + secretManager.verifyToken(id, token.getPassword()); + } finally { + dis.close(); + } + return id.getUser(); + } + + /** + * Create a timeline secret manager + * + * @param delegationKeyUpdateInterval + * the number of seconds for rolling new secret keys. + * @param delegationTokenMaxLifetime + * the maximum lifetime of the delegation tokens + * @param delegationTokenRenewInterval + * how often the tokens must be renewed + * @param delegationTokenRemoverScanInterval + * how often the tokens are scanned for expired tokens + */ + @Private + @Unstable + public static class TimelineDelegationTokenSecretManager extends + AbstractDelegationTokenSecretManager { + + public TimelineDelegationTokenSecretManager(long delegationKeyUpdateInterval, + long delegationTokenMaxLifetime, long delegationTokenRenewInterval, + long delegationTokenRemoverScanInterval) { + super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, + delegationTokenRenewInterval, delegationTokenRemoverScanInterval); + } + + @Override + public TimelineDelegationTokenIdentifier createIdentifier() { + return new TimelineDelegationTokenIdentifier(); + } + + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index 93065b33681..c3a19d476b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -19,25 +19,70 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; import static org.apache.hadoop.yarn.util.StringHelper.pajoin; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.YarnWebParams; +import com.google.common.annotations.VisibleForTesting; + public class AHSWebApp extends WebApp implements YarnWebParams { - private final ApplicationHistoryManager applicationHistoryManager; - private final TimelineStore timelineStore; + private ApplicationHistoryManager applicationHistoryManager; + private TimelineStore timelineStore; + private TimelineDelegationTokenSecretManagerService secretManagerService; - public AHSWebApp(ApplicationHistoryManager applicationHistoryManager, - TimelineStore timelineStore) { + private static AHSWebApp instance = null; + + public static AHSWebApp getInstance() { + if (instance == null) { + instance = new AHSWebApp(); + } + return instance; + } + + @Private + @VisibleForTesting + public static void resetInstance() { + instance = null; + } + + private AHSWebApp() { + + } + + public ApplicationHistoryManager getApplicationHistoryManager() { + return applicationHistoryManager; + } + + public void setApplicationHistoryManager( + ApplicationHistoryManager applicationHistoryManager) { this.applicationHistoryManager = applicationHistoryManager; + } + + public TimelineStore getTimelineStore() { + return timelineStore; + } + + public void setTimelineStore(TimelineStore timelineStore) { this.timelineStore = timelineStore; } + public TimelineDelegationTokenSecretManagerService + getTimelineDelegationTokenSecretManagerService() { + return secretManagerService; + } + + public void setTimelineDelegationTokenSecretManagerService( + TimelineDelegationTokenSecretManagerService secretManagerService) { + this.secretManagerService = secretManagerService; + } + @Override public void setup() { bind(YarnJacksonJaxbJsonProvider.class); @@ -46,6 +91,8 @@ public class AHSWebApp extends WebApp implements YarnWebParams { bind(GenericExceptionHandler.class); bind(ApplicationContext.class).toInstance(applicationHistoryManager); bind(TimelineStore.class).toInstance(timelineStore); + bind(TimelineDelegationTokenSecretManagerService.class).toInstance( + secretManagerService); route("/", AHSController.class); route(pajoin("/apps", APP_STATE), AHSController.class); route(pajoin("/app", APPLICATION_ID), AHSController.class, "app"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index a35fe4600f9..3f3c08a55df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.junit.After; import org.junit.Before; @@ -74,6 +75,7 @@ public class TestApplicationHistoryClientService extends @After public void tearDown() throws Exception { + AHSWebApp.resetInstance(); historyServer.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java index d6d20af189c..5c55becb6c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.junit.After; import org.junit.Test; @@ -40,7 +41,7 @@ public class TestApplicationHistoryServer { Configuration config = new YarnConfiguration(); historyServer.init(config); assertEquals(STATE.INITED, historyServer.getServiceState()); - assertEquals(3, historyServer.getServices().size()); + assertEquals(4, historyServer.getServices().size()); ApplicationHistoryClientService historyService = historyServer.getClientService(); assertNotNull(historyServer.getClientService()); @@ -73,5 +74,6 @@ public class TestApplicationHistoryServer { if (historyServer != null) { historyServer.stop(); } + AHSWebApp.resetInstance(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 2baddf7b198..93d0a125f94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistor import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; @@ -719,6 +720,7 @@ public class MiniYARNCluster extends CompositeService { if (appHistoryServer != null) { appHistoryServer.stop(); } + AHSWebApp.resetInstance(); super.serviceStop(); } } From af8b09af75b252aa75b86ca42323c5b21eb41ee2 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 23 May 2014 18:47:57 +0000 Subject: [PATCH 05/34] YARN-1936. Added security support for the Timeline Client. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597153 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/client/api/TimelineClient.java | 20 ++ .../client/api/impl/TimelineClientImpl.java | 180 +++++++++++++++++- .../yarn/client/api/impl/YarnClientImpl.java | 80 +++++++- .../yarn/client/api/impl/TestYarnClient.java | 91 ++++++++- .../yarn/util/timeline/TimelineUtils.java | 27 +++ ...neDelegationTokenSecretManagerService.java | 13 +- 7 files changed, 397 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 93764e67e84..359b4ed2ee7 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -93,6 +93,9 @@ Release 2.5.0 - UNRELEASED YARN-2049. Added delegation-token support for the Timeline Server. (Zhijie Shen via vinodkv) + YARN-1936. Added security support for the Timeline Client. (Zhijie Shen via + vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java index a2ed3e70a51..de1d3e2ae53 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java @@ -23,11 +23,13 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; /** * A client library that can be used to post some information in terms of a @@ -65,4 +67,22 @@ public abstract class TimelineClient extends AbstractService { public abstract TimelinePutResponse putEntities( TimelineEntity... entities) throws IOException, YarnException; + /** + *

+ * Get a delegation token so as to be able to talk to the timeline server in a + * secure way. + *

+ * + * @param renewer + * Address of the renewer who can renew these tokens when needed by + * securely talking to the timeline server + * @return a delegation token ({@link Token}) that can be used to talk to the + * timeline server + * @throws IOException + * @throws YarnException + */ + @Public + public abstract Token getDelegationToken( + String renewer) throws IOException, YarnException; + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 64cc041aaea..5ffe17a24a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -18,24 +18,43 @@ package org.apache.hadoop.yarn.client.api.impl; +import java.io.File; import java.io.IOException; +import java.net.HttpURLConnection; import java.net.URI; +import java.net.URL; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import javax.ws.rs.core.MediaType; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenSelector; +import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.codehaus.jackson.map.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; @@ -44,6 +63,8 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; +import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; @Private @Unstable @@ -52,16 +73,29 @@ public class TimelineClientImpl extends TimelineClient { private static final Log LOG = LogFactory.getLog(TimelineClientImpl.class); private static final String RESOURCE_URI_STR = "/ws/v1/timeline/"; private static final Joiner JOINER = Joiner.on(""); + private static Options opts; + static { + opts = new Options(); + opts.addOption("put", true, "Put the TimelineEntities in a JSON file"); + opts.getOption("put").setArgName("Path to the JSON file"); + opts.addOption("help", false, "Print usage"); + } private Client client; private URI resURI; private boolean isEnabled; + private TimelineAuthenticatedURLConnectionFactory urlFactory; public TimelineClientImpl() { super(TimelineClientImpl.class.getName()); ClientConfig cc = new DefaultClientConfig(); cc.getClasses().add(YarnJacksonJaxbJsonProvider.class); - client = Client.create(cc); + if (UserGroupInformation.isSecurityEnabled()) { + urlFactory = new TimelineAuthenticatedURLConnectionFactory(); + client = new Client(new URLConnectionClientHandler(urlFactory), cc); + } else { + client = Client.create(cc); + } } protected void serviceInit(Configuration conf) throws Exception { @@ -83,6 +117,9 @@ public class TimelineClientImpl extends TimelineClient { YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS), RESOURCE_URI_STR)); } + if (UserGroupInformation.isSecurityEnabled()) { + urlFactory.setService(TimelineUtils.buildTimelineTokenService(conf)); + } LOG.info("Timeline service address: " + resURI); } super.serviceInit(conf); @@ -124,6 +161,13 @@ public class TimelineClientImpl extends TimelineClient { return resp.getEntity(TimelinePutResponse.class); } + @Override + public Token getDelegationToken( + String renewer) throws IOException, YarnException { + return TimelineAuthenticator.getDelegationToken(resURI.toURL(), + urlFactory.token, renewer); + } + @Private @VisibleForTesting public ClientResponse doPostingEntities(TimelineEntities entities) { @@ -133,4 +177,138 @@ public class TimelineClientImpl extends TimelineClient { .post(ClientResponse.class, entities); } + private static class TimelineAuthenticatedURLConnectionFactory + implements HttpURLConnectionFactory { + + private AuthenticatedURL.Token token; + private TimelineAuthenticator authenticator; + private Token dToken; + private Text service; + + public TimelineAuthenticatedURLConnectionFactory() { + token = new AuthenticatedURL.Token(); + authenticator = new TimelineAuthenticator(); + } + + @Override + public HttpURLConnection getHttpURLConnection(URL url) throws IOException { + try { + if (dToken == null) { + //TODO: need to take care of the renew case + dToken = selectToken(); + if (LOG.isDebugEnabled()) { + LOG.debug("Timeline delegation token: " + dToken.toString()); + } + } + if (dToken != null) { + Map params = new HashMap(); + TimelineAuthenticator.injectDelegationToken(params, dToken); + url = TimelineAuthenticator.appendParams(url, params); + if (LOG.isDebugEnabled()) { + LOG.debug("URL with delegation token: " + url); + } + } + return new AuthenticatedURL(authenticator).openConnection(url, token); + } catch (AuthenticationException e) { + LOG.error("Authentication failed when openning connection [" + url + + "] with token [" + token + "].", e); + throw new IOException(e); + } + } + + private Token selectToken() { + UserGroupInformation ugi; + try { + ugi = UserGroupInformation.getCurrentUser(); + } catch (IOException e) { + String msg = "Error when getting the current user"; + LOG.error(msg, e); + throw new YarnRuntimeException(msg, e); + } + TimelineDelegationTokenSelector tokenSelector = + new TimelineDelegationTokenSelector(); + return tokenSelector.selectToken( + service, ugi.getCredentials().getAllTokens()); + } + + public void setService(Text service) { + this.service = service; + } + + } + + public static void main(String[] argv) throws Exception { + CommandLine cliParser = new GnuParser().parse(opts, argv); + if (cliParser.hasOption("put")) { + String path = cliParser.getOptionValue("put"); + if (path != null && path.length() > 0) { + putTimelineEntitiesInJSONFile(path); + return; + } + } + printUsage(); + } + + /** + * Put timeline data in a JSON file via command line. + * + * @param path + * path to the {@link TimelineEntities} JSON file + */ + private static void putTimelineEntitiesInJSONFile(String path) { + File jsonFile = new File(path); + if (!jsonFile.exists()) { + System.out.println("Error: File [" + jsonFile.getAbsolutePath() + + "] doesn't exist"); + return; + } + ObjectMapper mapper = new ObjectMapper(); + YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); + TimelineEntities entities = null; + try { + entities = mapper.readValue(jsonFile, TimelineEntities.class); + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + e.printStackTrace(System.err); + return; + } + Configuration conf = new YarnConfiguration(); + TimelineClient client = TimelineClient.createTimelineClient(); + client.init(conf); + client.start(); + try { + if (UserGroupInformation.isSecurityEnabled() + && conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) { + Token token = + client.getDelegationToken( + UserGroupInformation.getCurrentUser().getUserName()); + UserGroupInformation.getCurrentUser().addToken(token); + } + TimelinePutResponse response = client.putEntities( + entities.getEntities().toArray( + new TimelineEntity[entities.getEntities().size()])); + if (response.getErrors().size() == 0) { + System.out.println("Timeline data is successfully put"); + } else { + for (TimelinePutResponse.TimelinePutError error : response.getErrors()) { + System.out.println("TimelineEntity [" + error.getEntityType() + ":" + + error.getEntityId() + "] is not successfully put. Error code: " + + error.getErrorCode()); + } + } + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + e.printStackTrace(System.err); + } finally { + client.stop(); + } + } + + /** + * Helper function to print out usage + */ + private static void printUsage() { + new HelpFormatter().printHelp("TimelineClient", opts); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 8a0348b3368..f1a3b6eecea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.client.api.impl; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -29,8 +30,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.DataInputByteBuffer; +import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; @@ -64,6 +70,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; @@ -74,6 +81,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.api.AHSClient; +import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -82,8 +90,10 @@ import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; +import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; @@ -97,8 +107,11 @@ public class YarnClientImpl extends YarnClient { protected long submitPollIntervalMillis; private long asyncApiPollIntervalMillis; private long asyncApiPollTimeoutMillis; - protected AHSClient historyClient; + private AHSClient historyClient; private boolean historyServiceEnabled; + protected TimelineClient timelineClient; + protected Text timelineService; + protected boolean timelineServiceEnabled; private static final String ROOT = "root"; @@ -126,10 +139,17 @@ public class YarnClientImpl extends YarnClient { if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { historyServiceEnabled = true; - historyClient = AHSClientImpl.createAHSClient(); - historyClient.init(getConfig()); + historyClient = AHSClient.createAHSClient(); + historyClient.init(conf); } + if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { + timelineServiceEnabled = true; + timelineClient = TimelineClient.createTimelineClient(); + timelineClient.init(conf); + timelineService = TimelineUtils.buildTimelineTokenService(conf); + } super.serviceInit(conf); } @@ -141,6 +161,9 @@ public class YarnClientImpl extends YarnClient { if (historyServiceEnabled) { historyClient.start(); } + if (timelineServiceEnabled) { + timelineClient.start(); + } } catch (IOException e) { throw new YarnRuntimeException(e); } @@ -155,6 +178,9 @@ public class YarnClientImpl extends YarnClient { if (historyServiceEnabled) { historyClient.stop(); } + if (timelineServiceEnabled) { + timelineClient.stop(); + } super.serviceStop(); } @@ -189,6 +215,12 @@ public class YarnClientImpl extends YarnClient { Records.newRecord(SubmitApplicationRequest.class); request.setApplicationSubmissionContext(appContext); + // Automatically add the timeline DT into the CLC + // Only when the security and the timeline service are both enabled + if (isSecurityEnabled() && timelineServiceEnabled) { + addTimelineDelegationToken(appContext.getAMContainerSpec()); + } + //TODO: YARN-1763:Handle RM failovers during the submitApplication call. rmClient.submitApplication(request); @@ -238,6 +270,48 @@ public class YarnClientImpl extends YarnClient { return applicationId; } + private void addTimelineDelegationToken( + ContainerLaunchContext clc) throws YarnException, IOException { + org.apache.hadoop.security.token.Token timelineDelegationToken = + timelineClient.getDelegationToken( + UserGroupInformation.getCurrentUser().getUserName()); + if (timelineDelegationToken == null) { + return; + } + Credentials credentials = new Credentials(); + DataInputByteBuffer dibb = new DataInputByteBuffer(); + ByteBuffer tokens = clc.getTokens(); + if (tokens != null) { + dibb.reset(tokens); + credentials.readTokenStorageStream(dibb); + tokens.rewind(); + } + // If the timeline delegation token is already in the CLC, no need to add + // one more + for (org.apache.hadoop.security.token.Token token : credentials + .getAllTokens()) { + TokenIdentifier tokenIdentifier = token.decodeIdentifier(); + if (tokenIdentifier instanceof TimelineDelegationTokenIdentifier) { + return; + } + } + credentials.addToken(timelineService, timelineDelegationToken); + if (LOG.isDebugEnabled()) { + LOG.debug("Add timline delegation token into credentials: " + + timelineDelegationToken); + } + DataOutputBuffer dob = new DataOutputBuffer(); + credentials.writeTokenStorageToStream(dob); + tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + clc.setTokens(tokens); + } + + @Private + @VisibleForTesting + protected boolean isSecurityEnabled() { + return UserGroupInformation.isSecurityEnabled(); + } + @Override public void killApplication(ApplicationId applicationId) throws YarnException, IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index cfee6f78d0c..6407f7a1089 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -25,19 +25,26 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Set; -import org.junit.Assert; - import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.DataInputByteBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; @@ -69,19 +76,23 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.util.Records; +import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import org.junit.Assert; import org.junit.Test; public class TestYarnClient { @@ -725,4 +736,80 @@ public class TestYarnClient { IOUtils.closeQuietly(client); } } + + @Test + public void testAutomaticTimelineDelegationTokenLoading() + throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); + final Token dToken = + new Token(); + // crate a mock client + YarnClientImpl client = new YarnClientImpl() { + @Override + protected void serviceInit(Configuration conf) throws Exception { + if (getConfig().getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { + timelineServiceEnabled = true; + timelineClient = mock(TimelineClient.class); + when(timelineClient.getDelegationToken(any(String.class))) + .thenReturn(dToken); + timelineClient.init(getConfig()); + timelineService = TimelineUtils.buildTimelineTokenService(getConfig()); + } + this.setConfig(conf); + } + + @Override + protected void serviceStart() throws Exception { + rmClient = mock(ApplicationClientProtocol.class); + } + + @Override + protected void serviceStop() throws Exception { + } + + @Override + public ApplicationReport getApplicationReport(ApplicationId appId) { + ApplicationReport report = mock(ApplicationReport.class); + when(report.getYarnApplicationState()) + .thenReturn(YarnApplicationState.SUBMITTED); + return report; + } + + @Override + public boolean isSecurityEnabled() { + return true; + } + }; + client.init(conf); + client.start(); + ApplicationSubmissionContext context = + mock(ApplicationSubmissionContext.class); + ApplicationId applicationId = ApplicationId.newInstance(0, 1); + when(context.getApplicationId()).thenReturn(applicationId); + DataOutputBuffer dob = new DataOutputBuffer(); + Credentials credentials = new Credentials(); + credentials.writeTokenStorageToStream(dob); + ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + ContainerLaunchContext clc = ContainerLaunchContext.newInstance( + null, null, null, null, tokens, null); + when(context.getAMContainerSpec()).thenReturn(clc); + client.submitApplication(context); + // Check whether token is added or not + credentials = new Credentials(); + DataInputByteBuffer dibb = new DataInputByteBuffer(); + tokens = clc.getTokens(); + if (tokens != null) { + dibb.reset(tokens); + credentials.readTokenStorageStream(dibb); + tokens.rewind(); + } + Collection> dTokens = + credentials.getAllTokens(); + Assert.assertEquals(1, dTokens.size()); + Assert.assertEquals(dToken, dTokens.iterator().next()); + client.stop(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java index a62ed4869da..02b5eb4eabd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java @@ -19,9 +19,14 @@ package org.apache.hadoop.yarn.util.timeline; import java.io.IOException; +import java.net.InetSocketAddress; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.map.JsonMappingException; @@ -78,4 +83,26 @@ public class TimelineUtils { } } + public static InetSocketAddress getTimelineTokenServiceAddress( + Configuration conf) { + InetSocketAddress timelineServiceAddr = null; + if (YarnConfiguration.useHttps(conf)) { + timelineServiceAddr = conf.getSocketAddr( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT); + } else { + timelineServiceAddr = conf.getSocketAddr( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT); + } + return timelineServiceAddr; + } + + public static Text buildTimelineTokenService(Configuration conf) { + InetSocketAddress timelineServiceAddr = + getTimelineTokenServiceAddress(conf); + return SecurityUtil.buildTokenService(timelineServiceAddr); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java index fee9eb41cd8..2808dac60da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java @@ -34,6 +34,7 @@ import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecret import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.util.timeline.TimelineUtils; /** * The service wrapper of {@link TimelineDelegationTokenSecretManager} @@ -65,17 +66,7 @@ public class TimelineDelegationTokenSecretManagerService extends AbstractService 3600000); secretManager.startThreads(); - if (YarnConfiguration.useHttps(getConfig())) { - serviceAddr = getConfig().getSocketAddr( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT); - } else { - serviceAddr = getConfig().getSocketAddr( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT); - } + serviceAddr = TimelineUtils.getTimelineTokenServiceAddress(getConfig()); super.init(conf); } From 892ada8fb35d57a1120d548fdefb056824aa939f Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 23 May 2014 21:19:15 +0000 Subject: [PATCH 06/34] YARN-1937. Added owner-only ACLs support for Timeline Client and server. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597186 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../records/timeline/TimelinePutResponse.java | 11 + .../ApplicationHistoryServer.java | 8 + .../timeline/MemoryTimelineStore.java | 35 ++- .../timeline/TimelineStore.java | 21 +- .../security/TimelineACLsManager.java | 88 ++++++ .../webapp/AHSWebApp.java | 11 + .../webapp/TimelineWebServices.java | 183 +++++++++++- .../security/TestTimelineACLsManager.java | 85 ++++++ .../webapp/TestTimelineWebServices.java | 270 +++++++++++++++++- 10 files changed, 687 insertions(+), 28 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 359b4ed2ee7..76d5ecce898 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -96,6 +96,9 @@ Release 2.5.0 - UNRELEASED YARN-1936. Added security support for the Timeline Client. (Zhijie Shen via vinodkv) + YARN-1937. Added owner-only ACLs support for Timeline Client and server. + (Zhijie Shen via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java index 37c00466caa..77a97ba95fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java @@ -107,6 +107,17 @@ public class TimelinePutResponse { */ public static final int IO_EXCEPTION = 2; + /** + * Error code returned if the user specifies the timeline system reserved + * filter key + */ + public static final int SYSTEM_FILTER_CONFLICT = 3; + + /** + * Error code returned if the user is denied to access the timeline data + */ + public static final int ACCESS_DENIED = 4; + private String entityId; private String entityType; private int errorCode; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index a2f5a2489e9..5e1277f75fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineAuthenticationFilterInitializer; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; @@ -63,6 +64,7 @@ public class ApplicationHistoryServer extends CompositeService { protected ApplicationHistoryManager historyManager; protected TimelineStore timelineStore; protected TimelineDelegationTokenSecretManagerService secretManagerService; + protected TimelineACLsManager timelineACLsManager; protected WebApp webApp; public ApplicationHistoryServer() { @@ -79,6 +81,7 @@ public class ApplicationHistoryServer extends CompositeService { addIfService(timelineStore); secretManagerService = createTimelineDelegationTokenSecretManagerService(conf); addService(secretManagerService); + timelineACLsManager = createTimelineACLsManager(conf); DefaultMetricsSystem.initialize("ApplicationHistoryServer"); JvmMetrics.initSingleton("ApplicationHistoryServer", null); @@ -169,6 +172,10 @@ public class ApplicationHistoryServer extends CompositeService { return new TimelineDelegationTokenSecretManagerService(); } + protected TimelineACLsManager createTimelineACLsManager(Configuration conf) { + return new TimelineACLsManager(conf); + } + protected void startWebApp() { Configuration conf = getConfig(); // Play trick to make the customized filter will only be loaded by the @@ -196,6 +203,7 @@ public class ApplicationHistoryServer extends CompositeService { ahsWebApp.setApplicationHistoryManager(historyManager); ahsWebApp.setTimelineStore(timelineStore); ahsWebApp.setTimelineDelegationTokenSecretManagerService(secretManagerService); + ahsWebApp.setTimelineACLsManager(timelineACLsManager); webApp = WebApps .$for("applicationhistory", ApplicationHistoryClientService.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java index 86ac1f81107..06f3d607a00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -40,8 +39,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; /** @@ -314,15 +313,29 @@ public class MemoryTimelineStore entityToReturn.setEntityId(entity.getEntityId()); entityToReturn.setEntityType(entity.getEntityType()); entityToReturn.setStartTime(entity.getStartTime()); - entityToReturn.setEvents(fields.contains(Field.EVENTS) ? - entity.getEvents() : fields.contains(Field.LAST_EVENT_ONLY) ? - Arrays.asList(entity.getEvents().get(0)) : null); - entityToReturn.setRelatedEntities(fields.contains(Field.RELATED_ENTITIES) ? - entity.getRelatedEntities() : null); - entityToReturn.setPrimaryFilters(fields.contains(Field.PRIMARY_FILTERS) ? - entity.getPrimaryFilters() : null); - entityToReturn.setOtherInfo(fields.contains(Field.OTHER_INFO) ? - entity.getOtherInfo() : null); + // Deep copy + if (fields.contains(Field.EVENTS)) { + entityToReturn.addEvents(entity.getEvents()); + } else if (fields.contains(Field.LAST_EVENT_ONLY)) { + entityToReturn.addEvent(entity.getEvents().get(0)); + } else { + entityToReturn.setEvents(null); + } + if (fields.contains(Field.RELATED_ENTITIES)) { + entityToReturn.addRelatedEntities(entity.getRelatedEntities()); + } else { + entityToReturn.setRelatedEntities(null); + } + if (fields.contains(Field.PRIMARY_FILTERS)) { + entityToReturn.addPrimaryFilters(entity.getPrimaryFilters()); + } else { + entityToReturn.setPrimaryFilters(null); + } + if (fields.contains(Field.OTHER_INFO)) { + entityToReturn.addOtherInfo(entity.getOtherInfo()); + } else { + entityToReturn.setOtherInfo(null); + } return entityToReturn; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java index 6b50d832cfc..fc02873bebc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java @@ -18,12 +18,25 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public interface TimelineStore extends Service, TimelineReader, TimelineWriter { + + /** + * The system filter which will be automatically added to a + * {@link TimelineEntity}'s primary filter section when storing the entity. + * The filter key is case sensitive. Users are supposed not to use the key + * reserved by the timeline system. + */ + @Private + enum SystemFilter { + ENTITY_OWNER + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java new file mode 100644 index 00000000000..5bc8705222f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import java.io.IOException; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore.SystemFilter; + +import com.google.common.annotations.VisibleForTesting; + +/** + * TimelineACLsManager check the entity level timeline data access. + */ +@Private +public class TimelineACLsManager { + + private static final Log LOG = LogFactory.getLog(TimelineACLsManager.class); + + private boolean aclsEnabled; + + public TimelineACLsManager(Configuration conf) { + aclsEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, + YarnConfiguration.DEFAULT_YARN_ACL_ENABLE); + } + + public boolean checkAccess(UserGroupInformation callerUGI, + TimelineEntity entity) throws YarnException, IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Verifying the access of " + callerUGI.getShortUserName() + + " on the timeline entity " + + new EntityIdentifier(entity.getEntityId(), entity.getEntityType())); + } + + if (!aclsEnabled) { + return true; + } + + Set values = + entity.getPrimaryFilters().get( + SystemFilter.ENTITY_OWNER.toString()); + if (values == null || values.size() != 1) { + throw new YarnException("Owner information of the timeline entity " + + new EntityIdentifier(entity.getEntityId(), entity.getEntityType()) + + " is corrupted."); + } + String owner = values.iterator().next().toString(); + // TODO: Currently we just check the user is the timeline entity owner. In + // the future, we need to check whether the user is admin or is in the + // allowed user/group list + if (callerUGI != null && callerUGI.getShortUserName().equals(owner)) { + return true; + } + return false; + } + + @Private + @VisibleForTesting + public void setACLsEnabled(boolean aclsEnabled) { + this.aclsEnabled = aclsEnabled; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index c3a19d476b9..17b1e6286d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; @@ -36,6 +37,7 @@ public class AHSWebApp extends WebApp implements YarnWebParams { private ApplicationHistoryManager applicationHistoryManager; private TimelineStore timelineStore; private TimelineDelegationTokenSecretManagerService secretManagerService; + private TimelineACLsManager timelineACLsManager; private static AHSWebApp instance = null; @@ -83,6 +85,14 @@ public class AHSWebApp extends WebApp implements YarnWebParams { this.secretManagerService = secretManagerService; } + public TimelineACLsManager getTimelineACLsManager() { + return timelineACLsManager; + } + + public void setTimelineACLsManager(TimelineACLsManager timelineACLsManager) { + this.timelineACLsManager = timelineACLsManager; + } + @Override public void setup() { bind(YarnJacksonJaxbJsonProvider.class); @@ -93,6 +103,7 @@ public class AHSWebApp extends WebApp implements YarnWebParams { bind(TimelineStore.class).toInstance(timelineStore); bind(TimelineDelegationTokenSecretManagerService.class).toInstance( secretManagerService); + bind(TimelineACLsManager.class).toInstance(timelineACLsManager); route("/", AHSController.class); route(pajoin("/apps", APP_STATE), AHSController.class); route(pajoin("/app", APPLICATION_ID), AHSController.class, "app"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java index 44567ea229a..6041779b13a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.SortedSet; @@ -52,17 +53,21 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.NameValuePair; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; +import org.apache.hadoop.yarn.webapp.NotFoundException; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -75,10 +80,13 @@ public class TimelineWebServices { private static final Log LOG = LogFactory.getLog(TimelineWebServices.class); private TimelineStore store; + private TimelineACLsManager timelineACLsManager; @Inject - public TimelineWebServices(TimelineStore store) { + public TimelineWebServices(TimelineStore store, + TimelineACLsManager timelineACLsManager) { this.store = store; + this.timelineACLsManager = timelineACLsManager; } @XmlRootElement(name = "about") @@ -141,6 +149,9 @@ public class TimelineWebServices { init(res); TimelineEntities entities = null; try { + EnumSet fieldEnums = parseFieldsStr(fields, ","); + boolean modified = extendFields(fieldEnums); + UserGroupInformation callerUGI = getUser(req); entities = store.getEntities( parseStr(entityType), parseLongStr(limit), @@ -150,7 +161,33 @@ public class TimelineWebServices { parseLongStr(fromTs), parsePairStr(primaryFilter, ":"), parsePairsStr(secondaryFilter, ",", ":"), - parseFieldsStr(fields, ",")); + fieldEnums); + if (entities != null) { + Iterator entitiesItr = + entities.getEntities().iterator(); + while (entitiesItr.hasNext()) { + TimelineEntity entity = entitiesItr.next(); + try { + // check ACLs + if (!timelineACLsManager.checkAccess(callerUGI, entity)) { + entitiesItr.remove(); + } else { + // clean up system data + if (modified) { + entity.setPrimaryFilters(null); + } else { + cleanupOwnerInfo(entity); + } + } + } catch (YarnException e) { + LOG.error("Error when verifying access for user " + callerUGI + + " on the events of the timeline entity " + + new EntityIdentifier(entity.getEntityId(), + entity.getEntityType()), e); + entitiesItr.remove(); + } + } + } } catch (NumberFormatException e) { throw new BadRequestException( "windowStart, windowEnd or limit is not a numeric value."); @@ -182,9 +219,25 @@ public class TimelineWebServices { init(res); TimelineEntity entity = null; try { + EnumSet fieldEnums = parseFieldsStr(fields, ","); + boolean modified = extendFields(fieldEnums); entity = store.getEntity(parseStr(entityId), parseStr(entityType), - parseFieldsStr(fields, ",")); + fieldEnums); + if (entity != null) { + // check ACLs + UserGroupInformation callerUGI = getUser(req); + if (!timelineACLsManager.checkAccess(callerUGI, entity)) { + entity = null; + } else { + // clean up the system data + if (modified) { + entity.setPrimaryFilters(null); + } else { + cleanupOwnerInfo(entity); + } + } + } } catch (IllegalArgumentException e) { throw new BadRequestException( "requested invalid field."); @@ -192,9 +245,15 @@ public class TimelineWebServices { LOG.error("Error getting entity", e); throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); + } catch (YarnException e) { + LOG.error("Error getting entity", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); } if (entity == null) { - throw new WebApplicationException(Response.Status.NOT_FOUND); + throw new NotFoundException("Timeline entity " + + new EntityIdentifier(parseStr(entityId), parseStr(entityType)) + + " is not found"); } return entity; } @@ -217,6 +276,7 @@ public class TimelineWebServices { init(res); TimelineEvents events = null; try { + UserGroupInformation callerUGI = getUser(req); events = store.getEntityTimelines( parseStr(entityType), parseArrayStr(entityId, ","), @@ -224,6 +284,29 @@ public class TimelineWebServices { parseLongStr(windowStart), parseLongStr(windowEnd), parseArrayStr(eventType, ",")); + if (events != null) { + Iterator eventsItr = + events.getAllEvents().iterator(); + while (eventsItr.hasNext()) { + TimelineEvents.EventsOfOneEntity eventsOfOneEntity = eventsItr.next(); + try { + TimelineEntity entity = store.getEntity( + eventsOfOneEntity.getEntityId(), + eventsOfOneEntity.getEntityType(), + EnumSet.of(Field.PRIMARY_FILTERS)); + // check ACLs + if (!timelineACLsManager.checkAccess(callerUGI, entity)) { + eventsItr.remove(); + } + } catch (Exception e) { + LOG.error("Error when verifying access for user " + callerUGI + + " on the events of the timeline entity " + + new EntityIdentifier(eventsOfOneEntity.getEntityId(), + eventsOfOneEntity.getEntityType()), e); + eventsItr.remove(); + } + } + } } catch (NumberFormatException e) { throw new BadRequestException( "windowStart, windowEnd or limit is not a numeric value."); @@ -252,12 +335,61 @@ public class TimelineWebServices { if (entities == null) { return new TimelinePutResponse(); } + UserGroupInformation callerUGI = getUser(req); try { List entityIDs = new ArrayList(); + TimelineEntities entitiesToPut = new TimelineEntities(); + List errors = + new ArrayList(); for (TimelineEntity entity : entities.getEntities()) { EntityIdentifier entityID = new EntityIdentifier(entity.getEntityId(), entity.getEntityType()); + + // check if there is existing entity + try { + TimelineEntity existingEntity = + store.getEntity(entityID.getId(), entityID.getType(), + EnumSet.of(Field.PRIMARY_FILTERS)); + if (existingEntity != null + && !timelineACLsManager.checkAccess(callerUGI, existingEntity)) { + throw new YarnException("The timeline entity " + entityID + + " was not put by " + callerUGI + " before"); + } + } catch (Exception e) { + // Skip the entity which already exists and was put by others + LOG.warn("Skip the timeline entity: " + entityID + ", because " + + e.getMessage()); + TimelinePutResponse.TimelinePutError error = + new TimelinePutResponse.TimelinePutError(); + error.setEntityId(entityID.getId()); + error.setEntityType(entityID.getType()); + error.setErrorCode( + TimelinePutResponse.TimelinePutError.ACCESS_DENIED); + errors.add(error); + continue; + } + + // inject owner information for the access check + try { + injectOwnerInfo(entity, + callerUGI == null ? "" : callerUGI.getShortUserName()); + } catch (YarnException e) { + // Skip the entity which messes up the primary filter and record the + // error + LOG.warn("Skip the timeline entity: " + entityID + ", because " + + e.getMessage()); + TimelinePutResponse.TimelinePutError error = + new TimelinePutResponse.TimelinePutError(); + error.setEntityId(entityID.getId()); + error.setEntityType(entityID.getType()); + error.setErrorCode( + TimelinePutResponse.TimelinePutError.SYSTEM_FILTER_CONFLICT); + errors.add(error); + continue; + } + entityIDs.add(entityID); + entitiesToPut.addEntity(entity); if (LOG.isDebugEnabled()) { LOG.debug("Storing the entity " + entityID + ", JSON-style content: " + TimelineUtils.dumpTimelineRecordtoJSON(entity)); @@ -266,7 +398,10 @@ public class TimelineWebServices { if (LOG.isDebugEnabled()) { LOG.debug("Storing entities: " + CSV_JOINER.join(entityIDs)); } - return store.put(entities); + TimelinePutResponse response = store.put(entitiesToPut); + // add the errors of timeline system filter key conflict + response.addErrors(errors); + return response; } catch (IOException e) { LOG.error("Error putting entities", e); throw new WebApplicationException(e, @@ -350,6 +485,14 @@ public class TimelineWebServices { } } + private static boolean extendFields(EnumSet fieldEnums) { + boolean modified = false; + if (fieldEnums != null && !fieldEnums.contains(Field.PRIMARY_FILTERS)) { + fieldEnums.add(Field.PRIMARY_FILTERS); + modified = true; + } + return modified; + } private static Long parseLongStr(String str) { return str == null ? null : Long.parseLong(str.trim()); } @@ -358,4 +501,34 @@ public class TimelineWebServices { return str == null ? null : str.trim(); } + private static UserGroupInformation getUser(HttpServletRequest req) { + String remoteUser = req.getRemoteUser(); + UserGroupInformation callerUGI = null; + if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); + } + return callerUGI; + } + + private static void injectOwnerInfo(TimelineEntity timelineEntity, + String owner) throws YarnException { + if (timelineEntity.getPrimaryFilters() != null && + timelineEntity.getPrimaryFilters().containsKey( + TimelineStore.SystemFilter.ENTITY_OWNER)) { + throw new YarnException( + "User should not use the timeline system filter key: " + + TimelineStore.SystemFilter.ENTITY_OWNER); + } + timelineEntity.addPrimaryFilter( + TimelineStore.SystemFilter.ENTITY_OWNER + .toString(), owner); + } + + private static void cleanupOwnerInfo(TimelineEntity timelineEntity) { + if (timelineEntity.getPrimaryFilters() != null) { + timelineEntity.getPrimaryFilters().remove( + TimelineStore.SystemFilter.ENTITY_OWNER.toString()); + } + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java new file mode 100644 index 00000000000..2c536681aed --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.junit.Assert; +import org.junit.Test; + +public class TestTimelineACLsManager { + + @Test + public void testYarnACLsNotEnabled() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); + TimelineACLsManager timelineACLsManager = + new TimelineACLsManager(conf); + TimelineEntity entity = new TimelineEntity(); + entity.addPrimaryFilter( + TimelineStore.SystemFilter.ENTITY_OWNER + .toString(), "owner"); + Assert.assertTrue( + "Always true when ACLs are not enabled", + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("user"), entity)); + } + + @Test + public void testYarnACLsEnabled() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + TimelineACLsManager timelineACLsManager = + new TimelineACLsManager(conf); + TimelineEntity entity = new TimelineEntity(); + entity.addPrimaryFilter( + TimelineStore.SystemFilter.ENTITY_OWNER + .toString(), "owner"); + Assert.assertTrue( + "Owner should be allowed to access", + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("owner"), entity)); + Assert.assertFalse( + "Other shouldn't be allowed to access", + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("other"), entity)); + } + + @Test + public void testCorruptedOwnerInfo() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + TimelineACLsManager timelineACLsManager = + new TimelineACLsManager(conf); + TimelineEntity entity = new TimelineEntity(); + try { + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("owner"), entity); + Assert.fail("Exception is expected"); + } catch (YarnException e) { + Assert.assertTrue("It's not the exact expected exception", e.getMessage() + .contains("is corrupted.")); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java index 650184bd517..9b0ae3761d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java @@ -20,19 +20,32 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; import static org.junit.Assert.assertEquals; +import java.io.IOException; + +import javax.inject.Singleton; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; import javax.ws.rs.core.MediaType; -import org.junit.Assert; - +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TestMemoryTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.junit.Assert; import org.junit.Test; import com.google.inject.Guice; @@ -50,6 +63,8 @@ import com.sun.jersey.test.framework.WebAppDescriptor; public class TestTimelineWebServices extends JerseyTest { private static TimelineStore store; + private static TimelineACLsManager timelineACLsManager; + private static String remoteUser; private long beforeTime; private Injector injector = Guice.createInjector(new ServletModule() { @@ -65,7 +80,12 @@ public class TestTimelineWebServices extends JerseyTest { Assert.fail(); } bind(TimelineStore.class).toInstance(store); + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); + timelineACLsManager = new TimelineACLsManager(conf); + bind(TimelineACLsManager.class).toInstance(timelineACLsManager); serve("/*").with(GuiceContainer.class); + filter("/*").through(TestFilter.class); } }); @@ -340,8 +360,8 @@ public class TestTimelineWebServices extends JerseyTest { public void testPostEntities() throws Exception { TimelineEntities entities = new TimelineEntities(); TimelineEntity entity = new TimelineEntity(); - entity.setEntityId("test id"); - entity.setEntityType("test type"); + entity.setEntityId("test id 1"); + entity.setEntityType("test type 1"); entity.setStartTime(System.currentTimeMillis()); entities.addEntity(entity); WebResource r = resource(); @@ -355,14 +375,248 @@ public class TestTimelineWebServices extends JerseyTest { Assert.assertEquals(0, putResposne.getErrors().size()); // verify the entity exists in the store response = r.path("ws").path("v1").path("timeline") - .path("test type").path("test id") + .path("test type 1").path("test id 1") .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); entity = response.getEntity(TimelineEntity.class); Assert.assertNotNull(entity); - Assert.assertEquals("test id", entity.getEntityId()); - Assert.assertEquals("test type", entity.getEntityType()); + Assert.assertEquals("test id 1", entity.getEntityId()); + Assert.assertEquals("test type 1", entity.getEntityType()); } + @Test + public void testPostEntitiesWithYarnACLsEnabled() throws Exception { + timelineACLsManager.setACLsEnabled(true); + remoteUser = "tester"; + try { + TimelineEntities entities = new TimelineEntities(); + TimelineEntity entity = new TimelineEntity(); + entity.setEntityId("test id 2"); + entity.setEntityType("test type 2"); + entity.setStartTime(System.currentTimeMillis()); + entities.addEntity(entity); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class); + Assert.assertNotNull(putResponse); + Assert.assertEquals(0, putResponse.getErrors().size()); + + // override/append timeline data in the same entity with different user + remoteUser = "other"; + response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + putResponse = response.getEntity(TimelinePutResponse.class); + Assert.assertNotNull(putResponse); + Assert.assertEquals(1, putResponse.getErrors().size()); + Assert.assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED, + putResponse.getErrors().get(0).getErrorCode()); + } finally { + timelineACLsManager.setACLsEnabled(false); + remoteUser = null; + } + } + + @Test + public void testGetEntityWithYarnACLsEnabled() throws Exception { + timelineACLsManager.setACLsEnabled(true); + remoteUser = "tester"; + try { + TimelineEntities entities = new TimelineEntities(); + TimelineEntity entity = new TimelineEntity(); + entity.setEntityId("test id 3"); + entity.setEntityType("test type 3"); + entity.setStartTime(System.currentTimeMillis()); + entities.addEntity(entity); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + // verify the system data will not be exposed + // 1. No field specification + response = r.path("ws").path("v1").path("timeline") + .path("test type 3").path("test id 3") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + entity = response.getEntity(TimelineEntity.class); + Assert.assertNull(entity.getPrimaryFilters().get( + TimelineStore.SystemFilter.ENTITY_OWNER.toString())); + // 2. other field + response = r.path("ws").path("v1").path("timeline") + .path("test type 3").path("test id 3") + .queryParam("fields", "relatedentities") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + entity = response.getEntity(TimelineEntity.class); + Assert.assertNull(entity.getPrimaryFilters().get( + TimelineStore.SystemFilter.ENTITY_OWNER.toString())); + // 3. primaryfilters field + response = r.path("ws").path("v1").path("timeline") + .path("test type 3").path("test id 3") + .queryParam("fields", "primaryfilters") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + entity = response.getEntity(TimelineEntity.class); + Assert.assertNull(entity.getPrimaryFilters().get( + TimelineStore.SystemFilter.ENTITY_OWNER.toString())); + + // get entity with other user + remoteUser = "other"; + response = r.path("ws").path("v1").path("timeline") + .path("test type 3").path("test id 3") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + assertEquals(ClientResponse.Status.NOT_FOUND, + response.getClientResponseStatus()); + } finally { + timelineACLsManager.setACLsEnabled(false); + remoteUser = null; + } + } + + @Test + public void testGetEntitiesWithYarnACLsEnabled() { + timelineACLsManager.setACLsEnabled(true); + remoteUser = "tester"; + try { + TimelineEntities entities = new TimelineEntities(); + TimelineEntity entity = new TimelineEntity(); + entity.setEntityId("test id 4"); + entity.setEntityType("test type 4"); + entity.setStartTime(System.currentTimeMillis()); + entities.addEntity(entity); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + + remoteUser = "other"; + entities = new TimelineEntities(); + entity = new TimelineEntity(); + entity.setEntityId("test id 5"); + entity.setEntityType("test type 4"); + entity.setStartTime(System.currentTimeMillis()); + entities.addEntity(entity); + r = resource(); + response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + + response = r.path("ws").path("v1").path("timeline") + .path("test type 4") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + entities = response.getEntity(TimelineEntities.class); + assertEquals(1, entities.getEntities().size()); + assertEquals("test type 4", entities.getEntities().get(0).getEntityType()); + assertEquals("test id 5", entities.getEntities().get(0).getEntityId()); + } finally { + timelineACLsManager.setACLsEnabled(false); + remoteUser = null; + } + } + + @Test + public void testGetEventsWithYarnACLsEnabled() { + timelineACLsManager.setACLsEnabled(true); + remoteUser = "tester"; + try { + TimelineEntities entities = new TimelineEntities(); + TimelineEntity entity = new TimelineEntity(); + entity.setEntityId("test id 5"); + entity.setEntityType("test type 5"); + entity.setStartTime(System.currentTimeMillis()); + TimelineEvent event = new TimelineEvent(); + event.setEventType("event type 1"); + event.setTimestamp(System.currentTimeMillis()); + entity.addEvent(event); + entities.addEntity(entity); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + + remoteUser = "other"; + entities = new TimelineEntities(); + entity = new TimelineEntity(); + entity.setEntityId("test id 6"); + entity.setEntityType("test type 5"); + entity.setStartTime(System.currentTimeMillis()); + event = new TimelineEvent(); + event.setEventType("event type 2"); + event.setTimestamp(System.currentTimeMillis()); + entity.addEvent(event); + entities.addEntity(entity); + r = resource(); + response = r.path("ws").path("v1").path("timeline") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, entities); + + response = r.path("ws").path("v1").path("timeline") + .path("test type 5").path("events") + .queryParam("entityId", "test id 5,test id 6") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + TimelineEvents events = response.getEntity(TimelineEvents.class); + assertEquals(1, events.getAllEvents().size()); + assertEquals("test id 6", events.getAllEvents().get(0).getEntityId()); + } finally { + timelineACLsManager.setACLsEnabled(false); + remoteUser = null; + } + } + + @Singleton + private static class TestFilter implements Filter { + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain chain) throws IOException, ServletException { + if (request instanceof HttpServletRequest) { + request = + new TestHttpServletRequestWrapper((HttpServletRequest) request); + } + chain.doFilter(request, response); + } + + @Override + public void destroy() { + } + + } + + private static class TestHttpServletRequestWrapper extends HttpServletRequestWrapper { + + public TestHttpServletRequestWrapper(HttpServletRequest request) { + super(request); + } + + @Override + public String getRemoteUser() { + return TestTimelineWebServices.remoteUser; + } + + } } From 6c56612af54152d5f96acbdf1aa7f7954d1e6eae Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Fri, 23 May 2014 22:38:52 +0000 Subject: [PATCH 07/34] YARN-2012. Fair Scheduler: allow default queue placement rule to take an arbitrary queue (Ashwin Shankar via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597204 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../scheduler/fair/QueuePlacementRule.java | 25 ++++++++++--- .../fair/TestQueuePlacementPolicy.java | 35 +++++++++++++++++++ .../src/site/apt/FairScheduler.apt.vm | 5 +-- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 76d5ecce898..8bc5f4ea498 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -99,6 +99,9 @@ Release 2.5.0 - UNRELEASED YARN-1937. Added owner-only ACLs support for Timeline Client and server. (Zhijie Shen via vinodkv) + YARN-2012. Fair Scheduler: allow default queue placement rule to take an + arbitrary queue (Ashwin Shankar via Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java index fa50bb7d308..a16e35a3bb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java @@ -276,21 +276,38 @@ public abstract class QueuePlacementRule { } /** - * Places all apps in the default queue + * Places apps in the specified default queue. If no default queue is + * specified the app is placed in root.default queue. */ public static class Default extends QueuePlacementRule { + private String defaultQueueName; + + @Override + public void initializeFromXml(Element el) + throws AllocationConfigurationException { + defaultQueueName = el.getAttribute("queue"); + if (defaultQueueName != null && !defaultQueueName.isEmpty()) { + if (!defaultQueueName.startsWith("root.")) { + defaultQueueName = "root." + defaultQueueName; + } + } else { + defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; + } + super.initializeFromXml(el); + } + @Override protected String getQueueForApp(String requestedQueue, String user, Groups groups, Map> configuredQueues) { - return "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; + return defaultQueueName; } - + @Override public boolean isTerminal() { return true; } } - + /** * Rejects all apps */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java index 640d771f2c2..e20b0c33076 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java @@ -133,6 +133,22 @@ public class TestQueuePlacementPolicy { parse(sb.toString()); } + @Test + public void testDefaultRuleWithQueueAttribute() throws Exception { + // This test covers the use case where we would like default rule + // to point to a different queue by default rather than root.default + configuredQueues.get(FSQueueType.LEAF).add("root.someDefaultQueue"); + StringBuffer sb = new StringBuffer(); + sb.append(""); + sb.append(" "); + sb.append(" "); + sb.append(""); + + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.someDefaultQueue", + policy.assignAppToQueue("root.default", "user1")); + } + @Test public void testNestedUserQueueParsingErrors() { // No nested rule specified in hierarchical user queue @@ -311,6 +327,25 @@ public class TestQueuePlacementPolicy { policy.assignAppToQueue("root.parent2", "user2")); } + @Test + public void testNestedUserQueueDefaultRule() throws Exception { + // This test covers the use case where we would like user queues to be + // created under a default parent queue + configuredQueues.get(FSQueueType.PARENT).add("root.parentq"); + StringBuffer sb = new StringBuffer(); + sb.append(""); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(""); + + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.parentq.user1", + policy.assignAppToQueue("root.default", "user1")); + } + private QueuePlacementPolicy parse(String str) throws Exception { // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm index 02890a1c357..611c390a820 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm @@ -310,7 +310,8 @@ Allocation file format to ‘parent’ or by configuring at least one leaf under that queue which makes it a parent. See example allocation for a sample use case. - * default: the app is placed into the queue named "default". + * default: the app is placed into the queue specified in the ‘queue’ attribute of the + default rule. If ‘queue’ attribute is not specified, the app is placed into ‘root.default’ queue. * reject: the app is rejected. @@ -348,7 +349,7 @@ Allocation file format - + --- From 7f0724dddb75d24b91f67b581a08f89f1ecfadf0 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 23 May 2014 22:40:52 +0000 Subject: [PATCH 08/34] YARN-2059. Added admin ACLs support to Timeline Server. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597207 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../security/TimelineACLsManager.java | 24 ++++++++++-------- .../webapp/TimelineWebServices.java | 13 +++++++--- .../security/TestTimelineACLsManager.java | 6 +++++ .../webapp/TestTimelineWebServices.java | 25 +++++++++++++------ 5 files changed, 49 insertions(+), 22 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8bc5f4ea498..ba6f0c54f4c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -102,6 +102,9 @@ Release 2.5.0 - UNRELEASED YARN-2012. Fair Scheduler: allow default queue placement rule to take an arbitrary queue (Ashwin Shankar via Sandy Ryza) + YARN-2059. Added admin ACLs support to Timeline Server. (Zhijie Shen via + vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java index 5bc8705222f..8009b39c94f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java @@ -27,8 +27,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.AdminACLsManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore.SystemFilter; @@ -42,11 +42,10 @@ public class TimelineACLsManager { private static final Log LOG = LogFactory.getLog(TimelineACLsManager.class); - private boolean aclsEnabled; + private AdminACLsManager adminAclsManager; public TimelineACLsManager(Configuration conf) { - aclsEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, - YarnConfiguration.DEFAULT_YARN_ACL_ENABLE); + this.adminAclsManager = new AdminACLsManager(conf); } public boolean checkAccess(UserGroupInformation callerUGI, @@ -57,7 +56,7 @@ public class TimelineACLsManager { + new EntityIdentifier(entity.getEntityId(), entity.getEntityType())); } - if (!aclsEnabled) { + if (!adminAclsManager.areACLsEnabled()) { return true; } @@ -70,10 +69,12 @@ public class TimelineACLsManager { + " is corrupted."); } String owner = values.iterator().next().toString(); - // TODO: Currently we just check the user is the timeline entity owner. In - // the future, we need to check whether the user is admin or is in the + // TODO: Currently we just check the user is the admin or the timeline + // entity owner. In the future, we need to check whether the user is in the // allowed user/group list - if (callerUGI != null && callerUGI.getShortUserName().equals(owner)) { + if (callerUGI != null + && (adminAclsManager.isAdmin(callerUGI) || + callerUGI.getShortUserName().equals(owner))) { return true; } return false; @@ -81,8 +82,11 @@ public class TimelineACLsManager { @Private @VisibleForTesting - public void setACLsEnabled(boolean aclsEnabled) { - this.aclsEnabled = aclsEnabled; + public AdminACLsManager + setAdminACLsManager(AdminACLsManager adminAclsManager) { + AdminACLsManager oldAdminACLsManager = this.adminAclsManager; + this.adminAclsManager = adminAclsManager; + return oldAdminACLsManager; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java index 6041779b13a..5d749fa0906 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java @@ -346,8 +346,9 @@ public class TimelineWebServices { new EntityIdentifier(entity.getEntityId(), entity.getEntityType()); // check if there is existing entity + TimelineEntity existingEntity = null; try { - TimelineEntity existingEntity = + existingEntity = store.getEntity(entityID.getId(), entityID.getType(), EnumSet.of(Field.PRIMARY_FILTERS)); if (existingEntity != null @@ -369,10 +370,14 @@ public class TimelineWebServices { continue; } - // inject owner information for the access check + // inject owner information for the access check if this is the first + // time to post the entity, in case it's the admin who is updating + // the timeline data. try { - injectOwnerInfo(entity, - callerUGI == null ? "" : callerUGI.getShortUserName()); + if (existingEntity == null) { + injectOwnerInfo(entity, + callerUGI == null ? "" : callerUGI.getShortUserName()); + } } catch (YarnException e) { // Skip the entity which messes up the primary filter and record the // error diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java index 2c536681aed..39102b43bad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java @@ -49,6 +49,7 @@ public class TestTimelineACLsManager { public void testYarnACLsEnabled() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf); TimelineEntity entity = new TimelineEntity(); @@ -63,12 +64,17 @@ public class TestTimelineACLsManager { "Other shouldn't be allowed to access", timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("other"), entity)); + Assert.assertTrue( + "Admin should be allowed to access", + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("admin"), entity)); } @Test public void testCorruptedOwnerInfo() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner"); TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf); TimelineEntity entity = new TimelineEntity(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java index 9b0ae3761d2..7e3e4099403 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java @@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.AdminACLsManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TestMemoryTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; @@ -64,6 +65,7 @@ public class TestTimelineWebServices extends JerseyTest { private static TimelineStore store; private static TimelineACLsManager timelineACLsManager; + private static AdminACLsManager adminACLsManager; private static String remoteUser; private long beforeTime; @@ -83,6 +85,9 @@ public class TestTimelineWebServices extends JerseyTest { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); timelineACLsManager = new TimelineACLsManager(conf); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); + adminACLsManager = new AdminACLsManager(conf); bind(TimelineACLsManager.class).toInstance(timelineACLsManager); serve("/*").with(GuiceContainer.class); filter("/*").through(TestFilter.class); @@ -387,7 +392,8 @@ public class TestTimelineWebServices extends JerseyTest { @Test public void testPostEntitiesWithYarnACLsEnabled() throws Exception { - timelineACLsManager.setACLsEnabled(true); + AdminACLsManager oldAdminACLsManager = + timelineACLsManager.setAdminACLsManager(adminACLsManager); remoteUser = "tester"; try { TimelineEntities entities = new TimelineEntities(); @@ -419,14 +425,15 @@ public class TestTimelineWebServices extends JerseyTest { Assert.assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED, putResponse.getErrors().get(0).getErrorCode()); } finally { - timelineACLsManager.setACLsEnabled(false); + timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); remoteUser = null; } } @Test public void testGetEntityWithYarnACLsEnabled() throws Exception { - timelineACLsManager.setACLsEnabled(true); + AdminACLsManager oldAdminACLsManager = + timelineACLsManager.setAdminACLsManager(adminACLsManager); remoteUser = "tester"; try { TimelineEntities entities = new TimelineEntities(); @@ -481,14 +488,15 @@ public class TestTimelineWebServices extends JerseyTest { assertEquals(ClientResponse.Status.NOT_FOUND, response.getClientResponseStatus()); } finally { - timelineACLsManager.setACLsEnabled(false); + timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); remoteUser = null; } } @Test public void testGetEntitiesWithYarnACLsEnabled() { - timelineACLsManager.setACLsEnabled(true); + AdminACLsManager oldAdminACLsManager = + timelineACLsManager.setAdminACLsManager(adminACLsManager); remoteUser = "tester"; try { TimelineEntities entities = new TimelineEntities(); @@ -526,14 +534,15 @@ public class TestTimelineWebServices extends JerseyTest { assertEquals("test type 4", entities.getEntities().get(0).getEntityType()); assertEquals("test id 5", entities.getEntities().get(0).getEntityId()); } finally { - timelineACLsManager.setACLsEnabled(false); + timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); remoteUser = null; } } @Test public void testGetEventsWithYarnACLsEnabled() { - timelineACLsManager.setACLsEnabled(true); + AdminACLsManager oldAdminACLsManager = + timelineACLsManager.setAdminACLsManager(adminACLsManager); remoteUser = "tester"; try { TimelineEntities entities = new TimelineEntities(); @@ -579,7 +588,7 @@ public class TestTimelineWebServices extends JerseyTest { assertEquals(1, events.getAllEvents().size()); assertEquals("test id 6", events.getAllEvents().get(0).getEntityId()); } finally { - timelineACLsManager.setACLsEnabled(false); + timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); remoteUser = null; } } From a00b2d4f37518cb5e579ab7e49d0c8b33ded9cf4 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Fri, 23 May 2014 22:52:46 +0000 Subject: [PATCH 09/34] YARN-2073. Fair Scheduler: Add a utilization threshold to prevent preempting resources when cluster is free (Karthik Kambatla via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597209 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/FairScheduler.java | 32 +++- .../fair/FairSchedulerConfiguration.java | 8 + .../scheduler/fair/FairSchedulerTestBase.java | 172 +++++++++++++++++ .../scheduler/fair/TestFairScheduler.java | 145 +------------- .../fair/TestFairSchedulerPreemption.java | 178 ++++++++++++++++++ .../src/site/apt/FairScheduler.apt.vm | 6 + 7 files changed, 397 insertions(+), 147 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ba6f0c54f4c..ae2b1bab020 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -104,6 +104,9 @@ Release 2.5.0 - UNRELEASED YARN-2059. Added admin ACLs support to Timeline Server. (Zhijie Shen via vinodkv) + + YARN-2073. Fair Scheduler: Add a utilization threshold to prevent preempting + resources when cluster is free (Karthik Kambatla via Sandy Ryza) OPTIMIZATIONS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 5eeda64af0e..830f6f75099 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -148,7 +148,11 @@ public class FairScheduler extends // Time we last ran preemptTasksIfNecessary private long lastPreemptCheckTime; - // How often tasks are preempted + // Preemption related variables + protected boolean preemptionEnabled; + protected float preemptionUtilizationThreshold; + + // How often tasks are preempted protected long preemptionInterval; // ms to wait before force killing stuff (must be longer than a couple @@ -158,7 +162,6 @@ public class FairScheduler extends // Containers whose AMs have been warned that they will be preempted soon. private List warnedContainers = new ArrayList(); - protected boolean preemptionEnabled; protected boolean sizeBasedWeight; // Give larger weights to larger jobs protected WeightAdjuster weightAdjuster; // Can be null for no weight adjuster protected boolean continuousSchedulingEnabled; // Continuous Scheduling enabled or not @@ -318,7 +321,7 @@ public class FairScheduler extends * and then select the right ones using preemptTasks. */ protected synchronized void preemptTasksIfNecessary() { - if (!preemptionEnabled) { + if (!shouldAttemptPreemption()) { return; } @@ -328,10 +331,9 @@ public class FairScheduler extends } lastPreemptCheckTime = curTime; - Resource resToPreempt = Resources.none(); - + Resource resToPreempt = Resources.clone(Resources.none()); for (FSLeafQueue sched : queueMgr.getLeafQueues()) { - resToPreempt = Resources.add(resToPreempt, resToPreempt(sched, curTime)); + Resources.addTo(resToPreempt, resToPreempt(sched, curTime)); } if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, resToPreempt, Resources.none())) { @@ -1067,6 +1069,22 @@ public class FairScheduler extends clusterResource, rootMetrics.getAllocatedResources())); } + /** + * Check if preemption is enabled and the utilization threshold for + * preemption is met. + * + * @return true if preemption should be attempted, false otherwise. + */ + private boolean shouldAttemptPreemption() { + if (preemptionEnabled) { + return (preemptionUtilizationThreshold < Math.max( + (float) rootMetrics.getAvailableMB() / clusterResource.getMemory(), + (float) rootMetrics.getAvailableVirtualCores() / + clusterResource.getVirtualCores())); + } + return false; + } + @Override public QueueMetrics getRootQueueMetrics() { return rootMetrics; @@ -1172,6 +1190,8 @@ public class FairScheduler extends nodeLocalityDelayMs = this.conf.getLocalityDelayNodeMs(); rackLocalityDelayMs = this.conf.getLocalityDelayRackMs(); preemptionEnabled = this.conf.getPreemptionEnabled(); + preemptionUtilizationThreshold = + this.conf.getPreemptionUtilizationThreshold(); assignMultiple = this.conf.getAssignMultiple(); maxAssign = this.conf.getMaxAssign(); sizeBasedWeight = this.conf.getSizeBasedWeight(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index e271b053737..0fd242d662d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -101,6 +101,10 @@ public class FairSchedulerConfiguration extends Configuration { /** Whether preemption is enabled. */ protected static final String PREEMPTION = CONF_PREFIX + "preemption"; protected static final boolean DEFAULT_PREEMPTION = false; + + protected static final String PREEMPTION_THRESHOLD = + CONF_PREFIX + "preemption.cluster-utilization-threshold"; + protected static final float DEFAULT_PREEMPTION_THRESHOLD = 0.8f; protected static final String PREEMPTION_INTERVAL = CONF_PREFIX + "preemptionInterval"; protected static final int DEFAULT_PREEMPTION_INTERVAL = 5000; @@ -185,6 +189,10 @@ public class FairSchedulerConfiguration extends Configuration { return getBoolean(PREEMPTION, DEFAULT_PREEMPTION); } + public float getPreemptionUtilizationThreshold() { + return getFloat(PREEMPTION_THRESHOLD, DEFAULT_PREEMPTION_THRESHOLD); + } + public boolean getAssignMultiple() { return getBoolean(ASSIGN_MULTIPLE, DEFAULT_ASSIGN_MULTIPLE); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java new file mode 100644 index 00000000000..5f926763d9b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.Clock; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public class FairSchedulerTestBase { + protected static class MockClock implements Clock { + private long time = 0; + @Override + public long getTime() { + return time; + } + + public void tick(int seconds) { + time = time + seconds * 1000; + } + } + + protected final static String TEST_DIR = + new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); + + private static RecordFactory + recordFactory = RecordFactoryProvider.getRecordFactory(null); + + protected int APP_ID = 1; // Incrementing counter for scheduling apps + protected int ATTEMPT_ID = 1; // Incrementing counter for scheduling attempts + + protected Configuration conf; + protected FairScheduler scheduler; + protected ResourceManager resourceManager; + + // Helper methods + protected Configuration createConfiguration() { + Configuration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, + ResourceScheduler.class); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); + conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, + 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240); + conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, false); + conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f); + return conf; + } + + protected ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { + ApplicationId appIdImpl = ApplicationId.newInstance(0, appId); + return ApplicationAttemptId.newInstance(appIdImpl, attemptId); + } + + protected ResourceRequest createResourceRequest( + int memory, String host, int priority, int numContainers, + boolean relaxLocality) { + return createResourceRequest(memory, 1, host, priority, numContainers, + relaxLocality); + } + + protected ResourceRequest createResourceRequest( + int memory, int vcores, String host, int priority, int numContainers, + boolean relaxLocality) { + ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); + request.setCapability(BuilderUtils.newResource(memory, vcores)); + request.setResourceName(host); + request.setNumContainers(numContainers); + Priority prio = recordFactory.newRecordInstance(Priority.class); + prio.setPriority(priority); + request.setPriority(prio); + request.setRelaxLocality(relaxLocality); + return request; + } + + /** + * Creates a single container priority-1 request and submits to + * scheduler. + */ + protected ApplicationAttemptId createSchedulingRequest( + int memory, String queueId, String userId) { + return createSchedulingRequest(memory, queueId, userId, 1); + } + + protected ApplicationAttemptId createSchedulingRequest( + int memory, int vcores, String queueId, String userId) { + return createSchedulingRequest(memory, vcores, queueId, userId, 1); + } + + protected ApplicationAttemptId createSchedulingRequest( + int memory, String queueId, String userId, int numContainers) { + return createSchedulingRequest(memory, queueId, userId, numContainers, 1); + } + + protected ApplicationAttemptId createSchedulingRequest( + int memory, int vcores, String queueId, String userId, int numContainers) { + return createSchedulingRequest(memory, vcores, queueId, userId, numContainers, 1); + } + + protected ApplicationAttemptId createSchedulingRequest( + int memory, String queueId, String userId, int numContainers, int priority) { + return createSchedulingRequest(memory, 1, queueId, userId, numContainers, + priority); + } + + protected ApplicationAttemptId createSchedulingRequest( + int memory, int vcores, String queueId, String userId, int numContainers, + int priority) { + ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); + scheduler.addApplication(id.getApplicationId(), queueId, userId); + // This conditional is for testAclSubmitApplication where app is rejected + // and no app is added. + if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) { + scheduler.addApplicationAttempt(id, false); + } + List ask = new ArrayList(); + ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, + priority, numContainers, true); + ask.add(request); + scheduler.allocate(id, ask, new ArrayList(), null, null); + return id; + } + + protected void createSchedulingRequestExistingApplication( + int memory, int priority, ApplicationAttemptId attId) { + ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, + priority, 1, true); + createSchedulingRequestExistingApplication(request, attId); + } + + protected void createSchedulingRequestExistingApplication( + int memory, int vcores, int priority, ApplicationAttemptId attId) { + ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, + priority, 1, true); + createSchedulingRequestExistingApplication(request, attId); + } + + protected void createSchedulingRequestExistingApplication( + ResourceRequest request, ApplicationAttemptId attId) { + List ask = new ArrayList(); + ask.add(request); + scheduler.allocate(attId, ask, new ArrayList(), null, null); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index fe2cb2350e8..21c03a4257e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -55,7 +55,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; @@ -63,8 +62,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -94,7 +91,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateS import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Assert; @@ -105,46 +101,14 @@ import org.xml.sax.SAXException; import com.google.common.collect.Sets; @SuppressWarnings("unchecked") -public class TestFairScheduler { +public class TestFairScheduler extends FairSchedulerTestBase { + private final static String ALLOC_FILE = + new File(TEST_DIR, "test-queues").getAbsolutePath(); - static class MockClock implements Clock { - private long time = 0; - @Override - public long getTime() { - return time; - } - - public void tick(int seconds) { - time = time + seconds * 1000; - } - - } - - final static String TEST_DIR = new File(System.getProperty("test.build.data", - "/tmp")).getAbsolutePath(); - - final static String ALLOC_FILE = new File(TEST_DIR, - "test-queues").getAbsolutePath(); - - private FairScheduler scheduler; - private ResourceManager resourceManager; - private Configuration conf; - private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - - private int APP_ID = 1; // Incrementing counter for schedling apps - private int ATTEMPT_ID = 1; // Incrementing counter for scheduling attempts - - // HELPER METHODS @Before public void setUp() throws IOException { scheduler = new FairScheduler(); conf = createConfiguration(); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); - conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, - 1024); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240); - // All tests assume only one assignment per node update - conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); resourceManager = new ResourceManager(); resourceManager.init(conf); @@ -198,107 +162,6 @@ public class TestFairScheduler { } } - private Configuration createConfiguration() { - Configuration conf = new YarnConfiguration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, - ResourceScheduler.class); - return conf; - } - - private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { - ApplicationId appIdImpl = ApplicationId.newInstance(0, appId); - ApplicationAttemptId attId = - ApplicationAttemptId.newInstance(appIdImpl, attemptId); - return attId; - } - - private ResourceRequest createResourceRequest(int memory, String host, - int priority, int numContainers, boolean relaxLocality) { - return createResourceRequest(memory, 1, host, priority, numContainers, - relaxLocality); - } - - private ResourceRequest createResourceRequest(int memory, int vcores, String host, - int priority, int numContainers, boolean relaxLocality) { - ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); - request.setCapability(BuilderUtils.newResource(memory, vcores)); - request.setResourceName(host); - request.setNumContainers(numContainers); - Priority prio = recordFactory.newRecordInstance(Priority.class); - prio.setPriority(priority); - request.setPriority(prio); - request.setRelaxLocality(relaxLocality); - return request; - } - - /** - * Creates a single container priority-1 request and submits to - * scheduler. - */ - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, - String userId) { - return createSchedulingRequest(memory, queueId, userId, 1); - } - - private ApplicationAttemptId createSchedulingRequest(int memory, int vcores, - String queueId, String userId) { - return createSchedulingRequest(memory, vcores, queueId, userId, 1); - } - - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, - String userId, int numContainers) { - return createSchedulingRequest(memory, queueId, userId, numContainers, 1); - } - - private ApplicationAttemptId createSchedulingRequest(int memory, int vcores, - String queueId, String userId, int numContainers) { - return createSchedulingRequest(memory, vcores, queueId, userId, numContainers, 1); - } - - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, - String userId, int numContainers, int priority) { - return createSchedulingRequest(memory, 1, queueId, userId, numContainers, - priority); - } - - private ApplicationAttemptId createSchedulingRequest(int memory, int vcores, - String queueId, String userId, int numContainers, int priority) { - ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplication(id.getApplicationId(), queueId, userId); - // This conditional is for testAclSubmitApplication where app is rejected - // and no app is added. - if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) { - scheduler.addApplicationAttempt(id, false); - } - List ask = new ArrayList(); - ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, - priority, numContainers, true); - ask.add(request); - scheduler.allocate(id, ask, new ArrayList(), null, null); - return id; - } - - private void createSchedulingRequestExistingApplication(int memory, int priority, - ApplicationAttemptId attId) { - ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, - priority, 1, true); - createSchedulingRequestExistingApplication(request, attId); - } - - private void createSchedulingRequestExistingApplication(int memory, int vcores, - int priority, ApplicationAttemptId attId) { - ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, - priority, 1, true); - createSchedulingRequestExistingApplication(request, attId); - } - - private void createSchedulingRequestExistingApplication(ResourceRequest request, - ApplicationAttemptId attId) { - List ask = new ArrayList(); - ask.add(request); - scheduler.allocate(attId, ask, new ArrayList(), null, null); - } - // TESTS @Test(timeout=2000) @@ -1455,7 +1318,7 @@ public class TestFairScheduler { assertEquals( 1536, scheduler.resToPreempt(schedD, clock.getTime()).getMemory()); } - + @Test (timeout = 5000) public void testMultipleContainersWaitingForReservation() throws IOException { scheduler.reinitialize(conf, resourceManager.getRMContext()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java new file mode 100644 index 00000000000..b3ab299ea88 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Collection; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +public class TestFairSchedulerPreemption extends FairSchedulerTestBase { + private final static String ALLOC_FILE = new File(TEST_DIR, + TestFairSchedulerPreemption.class.getName() + ".xml").getAbsolutePath(); + + private MockClock clock; + + private static class StubbedFairScheduler extends FairScheduler { + public int lastPreemptMemory = -1; + + @Override + protected void preemptResources( + Collection scheds, Resource toPreempt) { + lastPreemptMemory = toPreempt.getMemory(); + } + + public void resetLastPreemptResources() { + lastPreemptMemory = -1; + } + } + + @Override + protected Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, StubbedFairScheduler.class, + ResourceScheduler.class); + conf.setBoolean(FairSchedulerConfiguration.PREEMPTION, true); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + return conf; + } + + @Before + public void setup() throws IOException { + conf = createConfiguration(); + clock = new MockClock(); + } + + @After + public void teardown() { + if (resourceManager != null) { + resourceManager.stop(); + resourceManager = null; + } + conf = null; + } + + private void startResourceManager(float utilizationThreshold) { + conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, + utilizationThreshold); + resourceManager = new MockRM(conf); + resourceManager.start(); + + assertTrue( + resourceManager.getResourceScheduler() instanceof StubbedFairScheduler); + scheduler = (FairScheduler)resourceManager.getResourceScheduler(); + + scheduler.setClock(clock); + scheduler.UPDATE_INTERVAL = 60 * 1000; + } + + private void registerNodeAndSubmitApp( + int memory, int vcores, int appContainers, int appMemory) { + RMNode node1 = MockNodes.newNodeInfo( + 1, Resources.createResource(memory, vcores), 1, "node1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + assertEquals("Incorrect amount of resources in the cluster", + memory, scheduler.rootMetrics.getAvailableMB()); + assertEquals("Incorrect amount of resources in the cluster", + vcores, scheduler.rootMetrics.getAvailableVirtualCores()); + + createSchedulingRequest(appMemory, "queueA", "user1", appContainers); + scheduler.update(); + // Sufficient node check-ins to fully schedule containers + for (int i = 0; i < 3; i++) { + NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1); + scheduler.handle(nodeUpdate1); + } + assertEquals("app1's request is not met", + memory - appContainers * appMemory, + scheduler.rootMetrics.getAvailableMB()); + } + + @Test + public void testPreemptionWithFreeResources() throws Exception { + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("0mb,0vcores"); + out.println(""); + out.println(""); + out.println("1"); + out.println("1024mb,0vcores"); + out.println(""); + out.println(""); + out.println("1"); + out.println("1024mb,0vcores"); + out.println(""); + out.print("5"); + out.print("10"); + out.println(""); + out.close(); + + startResourceManager(0f); + // Create node with 4GB memory and 4 vcores + registerNodeAndSubmitApp(4 * 1024, 4, 2, 1024); + + // Verify submitting another request doesn't trigger preemption + createSchedulingRequest(1024, "queueB", "user1", 1, 1); + scheduler.update(); + clock.tick(6); + + ((StubbedFairScheduler) scheduler).resetLastPreemptResources(); + scheduler.preemptTasksIfNecessary(); + assertEquals("preemptResources() should have been called", 1024, + ((StubbedFairScheduler) scheduler).lastPreemptMemory); + + resourceManager.stop(); + + startResourceManager(0.8f); + // Create node with 4GB memory and 4 vcores + registerNodeAndSubmitApp(4 * 1024, 4, 3, 1024); + + // Verify submitting another request doesn't trigger preemption + createSchedulingRequest(1024, "queueB", "user1", 1, 1); + scheduler.update(); + clock.tick(6); + + ((StubbedFairScheduler) scheduler).resetLastPreemptResources(); + scheduler.preemptTasksIfNecessary(); + assertEquals("preemptResources() should not have been called", -1, + ((StubbedFairScheduler) scheduler).lastPreemptMemory); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm index 611c390a820..48011cbe50e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm @@ -156,6 +156,12 @@ Properties that can be placed in yarn-site.xml * Whether to use preemption. Note that preemption is experimental in the current version. Defaults to false. + * <<>> + + * The utilization threshold after which preemption kicks in. The + utilization is computed as the maximum ratio of usage to capacity among + all resources. Defaults to 0.8f. + * <<>> * Whether to assign shares to individual apps based on their size, rather than From 7dd378c274bbbbc11f835a99fc5205f5c48c8faa Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Fri, 23 May 2014 23:51:00 +0000 Subject: [PATCH 10/34] YARN-2096. Race in TestRMRestart#testQueueMetricsOnRMRestart. (Anubhav Dhoot via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597223 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../server/resourcemanager/TestRMRestart.java | 15 +++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ae2b1bab020..c736f0ece0e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -143,6 +143,9 @@ Release 2.5.0 - UNRELEASED YARN-2089. FairScheduler: QueuePlacementPolicy and QueuePlacementRule are missing audience annotations. (Zhihai Xu via kasha) + YARN-2096. Race in TestRMRestart#testQueueMetricsOnRMRestart. + (Anubhav Dhoot via kasha) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 49eff8b0142..6a0512307a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -1638,10 +1638,16 @@ public class TestRMRestart { // create app that gets launched and does allocate before RM restart RMApp app1 = rm1.submitApp(200); - assertQueueMetrics(qm1, 1, 1, 0, 0); - nm1.nodeHeartbeat(true); + // Need to wait first for AppAttempt to be started (RMAppState.ACCEPTED) + // and then for it to reach RMAppAttemptState.SCHEDULED + // inorder to ensure appsPending metric is incremented + rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId(); + rm1.waitForState(attemptId1, RMAppAttemptState.SCHEDULED); + assertQueueMetrics(qm1, 1, 1, 0, 0); + + nm1.nodeHeartbeat(true); rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED); MockAM am1 = rm1.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); @@ -1660,11 +1666,12 @@ public class TestRMRestart { // PHASE 2: create new RM and start from old state // create new RM to represent restart and recover state MockRM rm2 = new MockRM(conf, memStore); - rm2.start(); - nm1.setResourceTrackerService(rm2.getResourceTrackerService()); QueueMetrics qm2 = rm2.getResourceScheduler().getRootQueueMetrics(); resetQueueMetrics(qm2); assertQueueMetrics(qm2, 0, 0, 0, 0); + + rm2.start(); + nm1.setResourceTrackerService(rm2.getResourceTrackerService()); // recover app RMApp loadedApp1 = rm2.getRMContext().getRMApps().get(app1.getApplicationId()); am1.setAMRMProtocol(rm2.getApplicationMasterService()); From 4dd2003373753f586fc7bc819932369314765f60 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sat, 24 May 2014 01:24:26 +0000 Subject: [PATCH 11/34] YARN-2071. Modified levelDB store permissions to be readable only by the server user. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597231 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../timeline/LeveldbTimelineStore.java | 45 +++++++++++++------ .../timeline/TestLeveldbTimelineStore.java | 20 +++++++-- 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c736f0ece0e..0827ef49ee0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -108,6 +108,9 @@ Release 2.5.0 - UNRELEASED YARN-2073. Fair Scheduler: Add a utilization threshold to prevent preempting resources when cluster is free (Karthik Kambatla via Sandy Ryza) + YARN-2071. Modified levelDB store permissions to be readable only by the + server user. (Zhijie Shen via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java index edd48426976..51bffda6274 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.readReverseOrderedLong; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; + import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -36,13 +39,16 @@ import java.util.TreeMap; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.google.common.annotations.VisibleForTesting; import org.apache.commons.collections.map.LRUMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.service.AbstractService; @@ -50,8 +56,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.fusesource.leveldbjni.JniDBFactory; @@ -62,8 +68,7 @@ import org.iq80.leveldb.ReadOptions; import org.iq80.leveldb.WriteBatch; import org.iq80.leveldb.WriteOptions; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.readReverseOrderedLong; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; +import com.google.common.annotations.VisibleForTesting; /** *

An implementation of an application timeline store backed by leveldb.

@@ -120,7 +125,9 @@ public class LeveldbTimelineStore extends AbstractService private static final Log LOG = LogFactory .getLog(LeveldbTimelineStore.class); - private static final String FILENAME = "leveldb-timeline-store.ldb"; + @Private + @VisibleForTesting + static final String FILENAME = "leveldb-timeline-store.ldb"; private static final byte[] START_TIME_LOOKUP_PREFIX = "k".getBytes(); private static final byte[] ENTITY_ENTRY_PREFIX = "e".getBytes(); @@ -135,6 +142,11 @@ public class LeveldbTimelineStore extends AbstractService private static final byte[] EMPTY_BYTES = new byte[0]; + @Private + @VisibleForTesting + static final FsPermission LEVELDB_DIR_UMASK = FsPermission + .createImmutable((short) 0700); + private Map startTimeWriteCache; private Map startTimeReadCache; @@ -164,16 +176,23 @@ public class LeveldbTimelineStore extends AbstractService YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE)); JniDBFactory factory = new JniDBFactory(); - String path = conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH); - File p = new File(path); - if (!p.exists()) { - if (!p.mkdirs()) { - throw new IOException("Couldn't create directory for leveldb " + - "timeline store " + path); + Path dbPath = new Path( + conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH), FILENAME); + FileSystem localFS = null; + try { + localFS = FileSystem.getLocal(conf); + if (!localFS.exists(dbPath)) { + if (!localFS.mkdirs(dbPath)) { + throw new IOException("Couldn't create directory for leveldb " + + "timeline store " + dbPath); + } + localFS.setPermission(dbPath, LEVELDB_DIR_UMASK); } + } finally { + IOUtils.cleanup(LOG, localFS); } - LOG.info("Using leveldb path " + path); - db = factory.open(new File(path, FILENAME), options); + LOG.info("Using leveldb path " + dbPath); + db = factory.open(new File(dbPath.toString()), options); startTimeWriteCache = Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize( conf))); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java index 9b273090bd0..59449f82dc1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import java.io.File; import java.io.IOException; import java.util.Collections; @@ -28,6 +32,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; @@ -39,9 +45,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; -import static org.junit.Assert.assertEquals; - @InterfaceAudience.Private @InterfaceStability.Unstable public class TestLeveldbTimelineStore extends TimelineStoreTestUtils { @@ -51,7 +54,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils { @Before public void setup() throws Exception { fsContext = FileContext.getLocalFSFileContext(); - Configuration conf = new Configuration(); + Configuration conf = new YarnConfiguration(); fsPath = new File("target", this.getClass().getSimpleName() + "-tmpDir").getAbsoluteFile(); fsContext.delete(new Path(fsPath.getAbsolutePath()), true); @@ -71,6 +74,15 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils { fsContext.delete(new Path(fsPath.getAbsolutePath()), true); } + @Test + public void testRootDirPermission() throws IOException { + FileSystem fs = FileSystem.getLocal(new YarnConfiguration()); + FileStatus file = fs.getFileStatus( + new Path(fsPath.getAbsolutePath(), LeveldbTimelineStore.FILENAME)); + assertNotNull(file); + assertEquals(LeveldbTimelineStore.LEVELDB_DIR_UMASK, file.getPermission()); + } + @Test public void testGetSingleEntity() throws IOException { super.testGetSingleEntity(); From 978e3a6813b142a4ab39e2205cc9d8ba37111a81 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sat, 24 May 2014 05:09:04 +0000 Subject: [PATCH 12/34] HDFS-6443. Fix MiniQJMHACluster related test failures. (Contributed by Zesheng Wu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597238 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/qjournal/MiniQJMHACluster.java | 90 +++++++++++-------- .../namenode/ha/TestFailureToReadEdits.java | 2 +- 3 files changed, 56 insertions(+), 39 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f0995e06582..385a07d1543 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -586,6 +586,9 @@ Release 2.5.0 - UNRELEASED HDFS-6423. Diskspace quota usage should be updated when appending data to partial block. (jing9) + HDFS-6443. Fix MiniQJMHACluster related test failures. (Zesheng Wu via + Arpit Agarwal) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java index e3b381c1f04..3166cccc94b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java @@ -22,8 +22,12 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import java.io.IOException; +import java.net.BindException; import java.net.URI; +import java.util.Random; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; @@ -37,14 +41,13 @@ public class MiniQJMHACluster { private MiniDFSCluster cluster; private MiniJournalCluster journalCluster; private final Configuration conf; + private static final Log LOG = LogFactory.getLog(MiniQJMHACluster.class); public static final String NAMESERVICE = "ns1"; private static final String NN1 = "nn1"; private static final String NN2 = "nn2"; - private static final int NN1_IPC_PORT = 10000; - private static final int NN1_INFO_PORT = 10001; - private static final int NN2_IPC_PORT = 10002; - private static final int NN2_INFO_PORT = 10003; + private static final Random RANDOM = new Random(); + private int basePort = 10000; public static class Builder { private final Configuration conf; @@ -69,51 +72,62 @@ public class MiniQJMHACluster { } } - public static MiniDFSNNTopology createDefaultTopology() { + public static MiniDFSNNTopology createDefaultTopology(int basePort) { return new MiniDFSNNTopology() .addNameservice(new MiniDFSNNTopology.NSConf(NAMESERVICE).addNN( - new MiniDFSNNTopology.NNConf("nn1").setIpcPort(NN1_IPC_PORT) - .setHttpPort(NN1_INFO_PORT)).addNN( - new MiniDFSNNTopology.NNConf("nn2").setIpcPort(NN2_IPC_PORT) - .setHttpPort(NN2_INFO_PORT))); + new MiniDFSNNTopology.NNConf("nn1").setIpcPort(basePort) + .setHttpPort(basePort + 1)).addNN( + new MiniDFSNNTopology.NNConf("nn2").setIpcPort(basePort + 2) + .setHttpPort(basePort + 3))); } - + private MiniQJMHACluster(Builder builder) throws IOException { this.conf = builder.conf; - // start 3 journal nodes - journalCluster = new MiniJournalCluster.Builder(conf).format(true) - .build(); - URI journalURI = journalCluster.getQuorumJournalURI(NAMESERVICE); - - // start cluster with 2 NameNodes - MiniDFSNNTopology topology = createDefaultTopology(); - - initHAConf(journalURI, builder.conf); - - // First start up the NNs just to format the namespace. The MinIDFSCluster - // has no way to just format the NameNodes without also starting them. - cluster = builder.dfsBuilder.nnTopology(topology) - .manageNameDfsSharedDirs(false).build(); - cluster.waitActive(); - cluster.shutdown(); - - // initialize the journal nodes - Configuration confNN0 = cluster.getConfiguration(0); - NameNode.initializeSharedEdits(confNN0, true); - - cluster.getNameNodeInfos()[0].setStartOpt(builder.startOpt); - cluster.getNameNodeInfos()[1].setStartOpt(builder.startOpt); - - // restart the cluster - cluster.restartNameNodes(); + int retryCount = 0; + while (true) { + try { + basePort = 10000 + RANDOM.nextInt(1000) * 4; + // start 3 journal nodes + journalCluster = new MiniJournalCluster.Builder(conf).format(true) + .build(); + URI journalURI = journalCluster.getQuorumJournalURI(NAMESERVICE); + + // start cluster with 2 NameNodes + MiniDFSNNTopology topology = createDefaultTopology(basePort); + + initHAConf(journalURI, builder.conf); + + // First start up the NNs just to format the namespace. The MinIDFSCluster + // has no way to just format the NameNodes without also starting them. + cluster = builder.dfsBuilder.nnTopology(topology) + .manageNameDfsSharedDirs(false).build(); + cluster.waitActive(); + cluster.shutdown(); + + // initialize the journal nodes + Configuration confNN0 = cluster.getConfiguration(0); + NameNode.initializeSharedEdits(confNN0, true); + + cluster.getNameNodeInfos()[0].setStartOpt(builder.startOpt); + cluster.getNameNodeInfos()[1].setStartOpt(builder.startOpt); + + // restart the cluster + cluster.restartNameNodes(); + ++retryCount; + break; + } catch (BindException e) { + LOG.info("MiniQJMHACluster port conflicts, retried " + + retryCount + " times"); + } + } } private Configuration initHAConf(URI journalURI, Configuration conf) { conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, journalURI.toString()); - String address1 = "127.0.0.1:" + NN1_IPC_PORT; - String address2 = "127.0.0.1:" + NN2_IPC_PORT; + String address1 = "127.0.0.1:" + basePort; + String address2 = "127.0.0.1:" + (basePort + 2); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, NAMESERVICE, NN1), address1); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java index 8bcc6d72a51..5ddc63574d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java @@ -104,7 +104,7 @@ public class TestFailureToReadEdits { HAUtil.setAllowStandbyReads(conf, true); if (clusterType == TestType.SHARED_DIR_HA) { - MiniDFSNNTopology topology = MiniQJMHACluster.createDefaultTopology(); + MiniDFSNNTopology topology = MiniQJMHACluster.createDefaultTopology(10000); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) .numDataNodes(0) From 1228f8f6fb16de4f0283dd1c7939e6fc3dfb7aae Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Mon, 26 May 2014 19:38:05 +0000 Subject: [PATCH 13/34] HBASE-6110 adding more slow action log in critical write path (Liang Xie via stack) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597633 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/DFSClient.java | 4 ++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 ++++- .../apache/hadoop/hdfs/DFSOutputStream.java | 21 ++++++++++- .../hdfs/server/datanode/BlockReceiver.java | 37 +++++++++++++++++-- .../hadoop/hdfs/server/datanode/DNConf.java | 13 ++++++- .../src/main/resources/hdfs-default.xml | 18 +++++++++ 7 files changed, 97 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 385a07d1543..6e5d3f5f791 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -317,6 +317,9 @@ Trunk (Unreleased) HDFS-6374. setXAttr should require the user to be the owner of the file or directory (Charles Lamb via wang) + HDFS-6110 adding more slow action log in critical write path + (Liang Xie via stack) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 5c7095931ad..4d2da799e15 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -276,6 +276,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory { final int retryTimesForGetLastBlockLength; final int retryIntervalForGetLastBlockLength; final long datanodeRestartTimeout; + final long dfsclientSlowIoWarningThresholdMs; final boolean useLegacyBlockReader; final boolean useLegacyBlockReaderLocal; @@ -430,6 +431,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory { datanodeRestartTimeout = conf.getLong( DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY, DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT) * 1000; + dfsclientSlowIoWarningThresholdMs = conf.getLong( + DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY, + DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT); } public boolean isUseLegacyBlockReaderLocal() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index e9a435873cc..5835f21131b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -645,5 +645,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_NFS_REGISTRATION_PORT_DEFAULT = 40; // Currently unassigned. public static final String DFS_NFS_ALLOW_INSECURE_PORTS_KEY = "dfs.nfs.allow.insecure.ports"; public static final boolean DFS_NFS_ALLOW_INSECURE_PORTS_DEFAULT = true; - + + // Slow io warning log threshold settings for dfsclient and datanode. + public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY = + "dfs.client.slow.io.warning.threshold.ms"; + public static final long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000; + public static final String DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY = + "dfs.datanode.slow.io.warning.threshold.ms"; + public static final long DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 300; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index e2bc316f025..8ae750d8e89 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -122,6 +122,7 @@ public class DFSOutputStream extends FSOutputSummer implements Syncable, CanSetDropBehind { private static final int MAX_PACKETS = 80; // each packet 64K, total 5MB private final DFSClient dfsClient; + private final long dfsclientSlowLogThresholdMs; private Socket s; // closed is accessed by different threads under different locks. private volatile boolean closed = false; @@ -788,11 +789,19 @@ public class DFSOutputStream extends FSOutputSummer // process responses from datanodes. try { // read an ack from the pipeline + long begin = Time.monotonicNow(); ack.readFields(blockReplyStream); - if (DFSClient.LOG.isDebugEnabled()) { + long duration = Time.monotonicNow() - begin; + if (duration > dfsclientSlowLogThresholdMs + && ack.getSeqno() != Packet.HEART_BEAT_SEQNO) { + DFSClient.LOG + .warn("Slow ReadProcessor read fields took " + duration + + "ms (threshold=" + dfsclientSlowLogThresholdMs + "ms); ack: " + + ack + ", targets: " + Arrays.asList(targets)); + } else if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("DFSClient " + ack); } - + long seqno = ack.getSeqno(); // processes response status from datanodes. for (int i = ack.getNumOfReplies()-1; i >=0 && dfsClient.clientRunning; i--) { @@ -1570,6 +1579,8 @@ public class DFSOutputStream extends FSOutputSummer } this.checksum = checksum; + this.dfsclientSlowLogThresholdMs = + dfsClient.getConf().dfsclientSlowIoWarningThresholdMs; } /** Construct a new output stream for creating a file. */ @@ -2001,6 +2012,7 @@ public class DFSOutputStream extends FSOutputSummer if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("Waiting for ack for: " + seqno); } + long begin = Time.monotonicNow(); try { synchronized (dataQueue) { while (!closed) { @@ -2020,6 +2032,11 @@ public class DFSOutputStream extends FSOutputSummer checkClosed(); } catch (ClosedChannelException e) { } + long duration = Time.monotonicNow() - begin; + if (duration > dfsclientSlowLogThresholdMs) { + DFSClient.LOG.warn("Slow waitForAckedSeqno took " + duration + + "ms (threshold=" + dfsclientSlowLogThresholdMs + "ms)"); + } } private synchronized void start() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index a7643b571e2..e36d40ccaed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -69,7 +69,7 @@ class BlockReceiver implements Closeable { @VisibleForTesting static long CACHE_DROP_LAG_BYTES = 8 * 1024 * 1024; - + private final long datanodeSlowLogThresholdMs; private DataInputStream in = null; // from where data are read private DataChecksum clientChecksum; // checksum used by client private DataChecksum diskChecksum; // checksum we write to disk @@ -140,7 +140,7 @@ class BlockReceiver implements Closeable { this.isDatanode = clientname.length() == 0; this.isClient = !this.isDatanode; this.restartBudget = datanode.getDnConf().restartReplicaExpiry; - + this.datanodeSlowLogThresholdMs = datanode.getDnConf().datanodeSlowIoWarningThresholdMs; //for datanode, we have //1: clientName.length() == 0, and //2: stage == null or PIPELINE_SETUP_CREATE @@ -335,6 +335,7 @@ class BlockReceiver implements Closeable { */ void flushOrSync(boolean isSync) throws IOException { long flushTotalNanos = 0; + long begin = Time.monotonicNow(); if (checksumOut != null) { long flushStartNanos = System.nanoTime(); checksumOut.flush(); @@ -363,6 +364,12 @@ class BlockReceiver implements Closeable { datanode.metrics.incrFsyncCount(); } } + long duration = Time.monotonicNow() - begin; + if (duration > datanodeSlowLogThresholdMs) { + LOG.warn("Slow flushOrSync took " + duration + "ms (threshold=" + + datanodeSlowLogThresholdMs + "ms), isSync:" + isSync + ", flushTotalNanos=" + + flushTotalNanos + "ns"); + } } /** @@ -488,8 +495,14 @@ class BlockReceiver implements Closeable { //First write the packet to the mirror: if (mirrorOut != null && !mirrorError) { try { + long begin = Time.monotonicNow(); packetReceiver.mirrorPacketTo(mirrorOut); mirrorOut.flush(); + long duration = Time.monotonicNow() - begin; + if (duration > datanodeSlowLogThresholdMs) { + LOG.warn("Slow BlockReceiver write packet to mirror took " + duration + + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)"); + } } catch (IOException e) { handleMirrorOutError(e); } @@ -572,7 +585,13 @@ class BlockReceiver implements Closeable { int numBytesToDisk = (int)(offsetInBlock-onDiskLen); // Write data to disk. + long begin = Time.monotonicNow(); out.write(dataBuf.array(), startByteToDisk, numBytesToDisk); + long duration = Time.monotonicNow() - begin; + if (duration > datanodeSlowLogThresholdMs) { + LOG.warn("Slow BlockReceiver write data to disk cost:" + duration + + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)"); + } // If this is a partial chunk, then verify that this is the only // chunk in the packet. Calculate new crc for this chunk. @@ -638,6 +657,7 @@ class BlockReceiver implements Closeable { try { if (outFd != null && offsetInBlock > lastCacheManagementOffset + CACHE_DROP_LAG_BYTES) { + long begin = Time.monotonicNow(); // // For SYNC_FILE_RANGE_WRITE, we want to sync from // lastCacheManagementOffset to a position "two windows ago" @@ -670,6 +690,11 @@ class BlockReceiver implements Closeable { NativeIO.POSIX.POSIX_FADV_DONTNEED); } lastCacheManagementOffset = offsetInBlock; + long duration = Time.monotonicNow() - begin; + if (duration > datanodeSlowLogThresholdMs) { + LOG.warn("Slow manageWriterOsCache took " + duration + + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)"); + } } } catch (Throwable t) { LOG.warn("Error managing cache for writer of block " + block, t); @@ -1299,9 +1324,15 @@ class BlockReceiver implements Closeable { replicaInfo.setBytesAcked(offsetInBlock); } // send my ack back to upstream datanode + long begin = Time.monotonicNow(); replyAck.write(upstreamOut); upstreamOut.flush(); - if (LOG.isDebugEnabled()) { + long duration = Time.monotonicNow() - begin; + if (duration > datanodeSlowLogThresholdMs) { + LOG.warn("Slow PacketResponder send ack to upstream took " + duration + + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), " + myString + + ", replyAck=" + replyAck); + } else if (LOG.isDebugEnabled()) { LOG.debug(myString + ", replyAck=" + replyAck); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java index d72c3b692ce..27dba1bacab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java @@ -79,6 +79,8 @@ public class DNConf { final long deleteReportInterval; final long initialBlockReportDelay; final long cacheReportInterval; + final long dfsclientSlowIoWarningThresholdMs; + final long datanodeSlowIoWarningThresholdMs; final int writePacketSize; final String minimumNameNodeVersion; @@ -129,7 +131,14 @@ public class DNConf { DFS_BLOCKREPORT_SPLIT_THRESHOLD_DEFAULT); this.cacheReportInterval = conf.getLong(DFS_CACHEREPORT_INTERVAL_MSEC_KEY, DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT); - + + this.dfsclientSlowIoWarningThresholdMs = conf.getLong( + DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY, + DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT); + this.datanodeSlowIoWarningThresholdMs = conf.getLong( + DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY, + DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT); + long initBRDelay = conf.getLong( DFS_BLOCKREPORT_INITIAL_DELAY_KEY, DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT) * 1000L; @@ -168,7 +177,7 @@ public class DNConf { DFS_DATANODE_RESTART_REPLICA_EXPIRY_KEY, DFS_DATANODE_RESTART_REPLICA_EXPIRY_DEFAULT) * 1000L; } - + // We get minimumNameNodeVersion via a method so it can be mocked out in tests. String getMinimumNameNodeVersion() { return this.minimumNameNodeVersion; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index e1ca17d5646..34b36a5c389 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -1930,4 +1930,22 @@ + + dfs.client.slow.io.warning.threshold.ms + 30000 + The threshold in milliseconds at which we will log a slow + io warning in a dfsclient. By default, this parameter is set to 30000 + milliseconds (30 seconds). + + + + + dfs.datanode.slow.io.warning.threshold.ms + 300 + The threshold in milliseconds at which we will log a slow + io warning in a datanode. By default, this parameter is set to 300 + milliseconds. + + + From 8d9e8cec9fd10fbeafa4ff38217f3984538b187a Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Tue, 27 May 2014 17:21:41 +0000 Subject: [PATCH 14/34] HDFS-6227. ShortCircuitCache#unref should purge ShortCircuitReplicas whose streams have been closed by java interrupts. Contributed by Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597829 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/shortcircuit/ShortCircuitCache.java | 21 ++- .../hadoop/hdfs/TestBlockReaderFactory.java | 120 ++++++++++++++++++ 3 files changed, 139 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6e5d3f5f791..2987e5184a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -592,6 +592,9 @@ Release 2.5.0 - UNRELEASED HDFS-6443. Fix MiniQJMHACluster related test failures. (Zesheng Wu via Arpit Agarwal) + HDFS-6227. ShortCircuitCache#unref should purge ShortCircuitReplicas whose + streams have been closed by java interrupts. (Colin Patrick McCabe via jing9) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index 1e7245df44b..7df340ac2a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -437,11 +437,22 @@ public class ShortCircuitCache implements Closeable { void unref(ShortCircuitReplica replica) { lock.lock(); try { - // If the replica is stale, but we haven't purged it yet, let's do that. - // It would be a shame to evict a non-stale replica so that we could put - // a stale one into the cache. - if ((!replica.purged) && replica.isStale()) { - purge(replica); + // If the replica is stale or unusable, but we haven't purged it yet, + // let's do that. It would be a shame to evict a non-stale replica so + // that we could put a stale or unusable one into the cache. + if (!replica.purged) { + String purgeReason = null; + if (!replica.getDataStream().getChannel().isOpen()) { + purgeReason = "purging replica because its data channel is closed."; + } else if (!replica.getMetaStream().getChannel().isOpen()) { + purgeReason = "purging replica because its meta channel is closed."; + } else if (replica.isStale()) { + purgeReason = "purging replica because it is stale."; + } + if (purgeReason != null) { + LOG.debug(this + ": " + purgeReason); + purge(replica); + } } String addedString = ""; boolean shouldTrimEvictionMaps = false; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java index 09d4353d7f1..a7dcbc1168c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java @@ -28,12 +28,15 @@ import static org.hamcrest.CoreMatchers.equalTo; import java.io.File; import java.io.IOException; +import java.nio.channels.ClosedByInterruptException; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -409,4 +412,121 @@ public class TestBlockReaderFactory { getDomainSocketWatcher().isClosed()); cluster.shutdown(); } + + /** + * When an InterruptedException is sent to a thread calling + * FileChannel#read, the FileChannel is immediately closed and the + * thread gets an exception. This effectively means that we might have + * someone asynchronously calling close() on the file descriptors we use + * in BlockReaderLocal. So when unreferencing a ShortCircuitReplica in + * ShortCircuitCache#unref, we should check if the FileChannel objects + * are still open. If not, we should purge the replica to avoid giving + * it out to any future readers. + * + * This is a regression test for HDFS-6227: Short circuit read failed + * due to ClosedChannelException. + * + * Note that you may still get ClosedChannelException errors if two threads + * are reading from the same replica and an InterruptedException is delivered + * to one of them. + */ + @Test(timeout=120000) + public void testPurgingClosedReplicas() throws Exception { + BlockReaderTestUtil.enableBlockReaderFactoryTracing(); + final AtomicInteger replicasCreated = new AtomicInteger(0); + final AtomicBoolean testFailed = new AtomicBoolean(false); + DFSInputStream.tcpReadsDisabledForTesting = true; + BlockReaderFactory.createShortCircuitReplicaInfoCallback = + new ShortCircuitCache.ShortCircuitReplicaCreator() { + @Override + public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { + replicasCreated.incrementAndGet(); + return null; + } + }; + TemporarySocketDirectory sockDir = new TemporarySocketDirectory(); + Configuration conf = createShortCircuitConf( + "testPurgingClosedReplicas", sockDir); + final MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 4095; + final int SEED = 0xFADE0; + final DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(0), conf); + DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + + final Semaphore sem = new Semaphore(0); + final List locatedBlocks = + cluster.getNameNode().getRpcServer().getBlockLocations( + TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks(); + final LocatedBlock lblock = locatedBlocks.get(0); // first block + final byte[] buf = new byte[TEST_FILE_LEN]; + Runnable readerRunnable = new Runnable() { + @Override + public void run() { + try { + while (true) { + BlockReader blockReader = null; + try { + blockReader = BlockReaderTestUtil. + getBlockReader(cluster, lblock, 0, TEST_FILE_LEN); + sem.release(); + try { + blockReader.readAll(buf, 0, TEST_FILE_LEN); + } finally { + sem.acquireUninterruptibly(); + } + } catch (ClosedByInterruptException e) { + LOG.info("got the expected ClosedByInterruptException", e); + sem.release(); + break; + } finally { + if (blockReader != null) blockReader.close(); + } + LOG.info("read another " + TEST_FILE_LEN + " bytes."); + } + } catch (Throwable t) { + LOG.error("getBlockReader failure", t); + testFailed.set(true); + sem.release(); + } + } + }; + Thread thread = new Thread(readerRunnable); + thread.start(); + + // While the thread is reading, send it interrupts. + // These should trigger a ClosedChannelException. + while (thread.isAlive()) { + sem.acquireUninterruptibly(); + thread.interrupt(); + sem.release(); + } + Assert.assertFalse(testFailed.get()); + + // We should be able to read from the file without + // getting a ClosedChannelException. + BlockReader blockReader = null; + try { + blockReader = BlockReaderTestUtil. + getBlockReader(cluster, lblock, 0, TEST_FILE_LEN); + blockReader.readFully(buf, 0, TEST_FILE_LEN); + } finally { + if (blockReader != null) blockReader.close(); + } + byte expected[] = DFSTestUtil. + calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); + Assert.assertTrue(Arrays.equals(buf, expected)); + + // Another ShortCircuitReplica object should have been created. + Assert.assertEquals(2, replicasCreated.get()); + + dfs.close(); + cluster.shutdown(); + sockDir.close(); + } } From 1c867b1de8a9f4c6bb118c08c7b714bd2b8356cd Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Tue, 27 May 2014 20:21:33 +0000 Subject: [PATCH 15/34] HDFS-6416. Use Time#monotonicNow in OpenFileCtx and OpenFileCtxCatch to avoid system clock bugs. Contributed by Abhiraj Butala git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597868 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 11 ++++++----- .../apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java | 9 +++++---- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java index 95dacd86aaa..ae97ffefc31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java @@ -54,6 +54,7 @@ import org.apache.hadoop.nfs.nfs3.response.WccData; import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.VerifierNone; import org.apache.hadoop.util.Daemon; +import org.apache.hadoop.util.Time; import org.jboss.netty.channel.Channel; import com.google.common.annotations.VisibleForTesting; @@ -136,7 +137,7 @@ class OpenFileCtx { this.channel = channel; this.xid = xid; this.preOpAttr = preOpAttr; - this.startTime = System.currentTimeMillis(); + this.startTime = Time.monotonicNow(); } @Override @@ -158,11 +159,11 @@ class OpenFileCtx { private Daemon dumpThread; private void updateLastAccessTime() { - lastAccessTime = System.currentTimeMillis(); + lastAccessTime = Time.monotonicNow(); } private boolean checkStreamTimeout(long streamTimeout) { - return System.currentTimeMillis() - lastAccessTime > streamTimeout; + return Time.monotonicNow() - lastAccessTime > streamTimeout; } long getLastAccessTime() { @@ -696,7 +697,7 @@ class OpenFileCtx { + " updating the mtime, then return success"); Nfs3FileAttributes postOpAttr = null; try { - dfsClient.setTimes(path, System.currentTimeMillis(), -1); + dfsClient.setTimes(path, Time.monotonicNow(), -1); postOpAttr = Nfs3Utils.getFileAttr(dfsClient, path, iug); } catch (IOException e) { LOG.info("Got error when processing perfect overwrite, path=" + path @@ -997,7 +998,7 @@ class OpenFileCtx { if (LOG.isDebugEnabled()) { LOG.debug("FileId: " + latestAttr.getFileId() + " Service time:" - + (System.currentTimeMillis() - commit.getStartTime()) + + (Time.monotonicNow() - commit.getStartTime()) + "ms. Sent response for commit:" + commit); } entry = pendingCommits.firstEntry(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java index d255e3d9b99..1c06cf40588 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.nfs.nfs3.FileHandle; import org.apache.hadoop.nfs.nfs3.Nfs3Constant; import org.apache.hadoop.util.Daemon; +import org.apache.hadoop.util.Time; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -99,7 +100,7 @@ class OpenFileCtxCache { LOG.warn("No eviction candidate. All streams have pending work."); return null; } else { - long idleTime = System.currentTimeMillis() + long idleTime = Time.monotonicNow() - idlest.getValue().getLastAccessTime(); if (idleTime < Nfs3Constant.OUTPUT_STREAM_TIMEOUT_MIN_DEFAULT) { if (LOG.isDebugEnabled()) { @@ -250,7 +251,7 @@ class OpenFileCtxCache { // Check if it can sleep try { - long workedTime = System.currentTimeMillis() - lastWakeupTime; + long workedTime = Time.monotonicNow() - lastWakeupTime; if (workedTime < rotation) { if (LOG.isTraceEnabled()) { LOG.trace("StreamMonitor can still have a sleep:" @@ -258,7 +259,7 @@ class OpenFileCtxCache { } Thread.sleep(rotation - workedTime); } - lastWakeupTime = System.currentTimeMillis(); + lastWakeupTime = Time.monotonicNow(); } catch (InterruptedException e) { LOG.info("StreamMonitor got interrupted"); @@ -267,4 +268,4 @@ class OpenFileCtxCache { } } } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2987e5184a9..d1b9f391d9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -446,6 +446,9 @@ Release 2.5.0 - UNRELEASED HDFS-6435. Add support for specifying a static uid/gid mapping for the NFS gateway. (atm via wang) + HDFS-6416. Use Time#monotonicNow in OpenFileCtx and OpenFileCtxCatch to + avoid system clock bugs (Abhiraj Butala via brandonli) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) From 0ec6fc9e3c59c474f45e0fa68bb511778070a13c Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Tue, 27 May 2014 22:58:55 +0000 Subject: [PATCH 16/34] HDFS-6411. nfs-hdfs-gateway mount raises I/O error and hangs when a unauthorized user attempts to access it. Contributed by Brandon Li git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597895 13f79535-47bb-0310-9956-ffa450edef68 --- .../nfs/nfs3/response/ACCESS3Response.java | 6 ++-- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 34 +++++++++++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/ACCESS3Response.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/ACCESS3Response.java index 8ba07aa5b37..5720fc64797 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/ACCESS3Response.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/ACCESS3Response.java @@ -46,10 +46,12 @@ public class ACCESS3Response extends NFS3Response { @Override public XDR writeHeaderAndResponse(XDR out, int xid, Verifier verifier) { super.writeHeaderAndResponse(out, xid, verifier); - out.writeBoolean(true); - postOpAttr.serialize(out); if (this.getStatus() == Nfs3Status.NFS3_OK) { + out.writeBoolean(true); + postOpAttr.serialize(out); out.writeInt(access); + } else { + out.writeBoolean(false); } return out; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java index f78598f13a2..62aa8cbedc0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java @@ -124,6 +124,7 @@ import org.apache.hadoop.oncrpc.security.VerifierNone; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AuthorizationException; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; import org.jboss.netty.channel.Channel; @@ -270,6 +271,17 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface { Nfs3FileAttributes attrs = null; try { attrs = writeManager.getFileAttr(dfsClient, handle, iug); + } catch (RemoteException r) { + LOG.warn("Exception ", r); + IOException io = r.unwrapRemoteException(); + /** + * AuthorizationException can be thrown if the user can't be proxy'ed. + */ + if (io instanceof AuthorizationException) { + return new GETATTR3Response(Nfs3Status.NFS3ERR_ACCES); + } else { + return new GETATTR3Response(Nfs3Status.NFS3ERR_IO); + } } catch (IOException e) { LOG.info("Can't get file attribute, fileId=" + handle.getFileId(), e); response.setStatus(Nfs3Status.NFS3ERR_IO); @@ -499,6 +511,17 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface { securityHandler.getUid(), securityHandler.getGid(), attrs); return new ACCESS3Response(Nfs3Status.NFS3_OK, attrs, access); + } catch (RemoteException r) { + LOG.warn("Exception ", r); + IOException io = r.unwrapRemoteException(); + /** + * AuthorizationException can be thrown if the user can't be proxy'ed. + */ + if (io instanceof AuthorizationException) { + return new ACCESS3Response(Nfs3Status.NFS3ERR_ACCES); + } else { + return new ACCESS3Response(Nfs3Status.NFS3ERR_IO); + } } catch (IOException e) { LOG.warn("Exception ", e); return new ACCESS3Response(Nfs3Status.NFS3ERR_IO); @@ -1680,6 +1703,17 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface { return new FSSTAT3Response(Nfs3Status.NFS3_OK, attrs, totalBytes, freeBytes, freeBytes, maxFsObjects, maxFsObjects, maxFsObjects, 0); + } catch (RemoteException r) { + LOG.warn("Exception ", r); + IOException io = r.unwrapRemoteException(); + /** + * AuthorizationException can be thrown if the user can't be proxy'ed. + */ + if (io instanceof AuthorizationException) { + return new FSSTAT3Response(Nfs3Status.NFS3ERR_ACCES); + } else { + return new FSSTAT3Response(Nfs3Status.NFS3ERR_IO); + } } catch (IOException e) { LOG.warn("Exception ", e); return new FSSTAT3Response(Nfs3Status.NFS3ERR_IO); diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d1b9f391d9f..5814821b0e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -681,6 +681,9 @@ Release 2.4.1 - UNRELEASED HDFS-6397. NN shows inconsistent value in deadnode count. (Mohammad Kamrul Islam via kihwal) + HDFS-6411. nfs-hdfs-gateway mount raises I/O error and hangs when a + unauthorized user attempts to access it (brandonli) + Release 2.4.0 - 2014-04-07 INCOMPATIBLE CHANGES From edfbc8ad4a009c04bf6ffb346f8b25b51c5345e3 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Tue, 27 May 2014 23:46:22 +0000 Subject: [PATCH 17/34] YARN-2105. Fix TestFairScheduler after YARN-2012. (Ashwin Shankar via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597902 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../scheduler/fair/QueuePlacementRule.java | 12 +++++++- .../scheduler/fair/TestFairScheduler.java | 30 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0827ef49ee0..81ff5bc7db8 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -149,6 +149,9 @@ Release 2.5.0 - UNRELEASED YARN-2096. Race in TestRMRestart#testQueueMetricsOnRMRestart. (Anubhav Dhoot via kasha) + YARN-2105. Fix TestFairScheduler after YARN-2012. (Ashwin Shankar via + Sandy Ryza) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java index a16e35a3bb3..056df577c7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java @@ -280,8 +280,18 @@ public abstract class QueuePlacementRule { * specified the app is placed in root.default queue. */ public static class Default extends QueuePlacementRule { - private String defaultQueueName; + @VisibleForTesting + String defaultQueueName; + @Override + public QueuePlacementRule initialize(boolean create, + Map args) { + if (defaultQueueName == null) { + defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; + } + return super.initialize(create, args); + } + @Override public void initializeFromXml(Element el) throws AllocationConfigurationException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 21c03a4257e..276020f00c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -88,6 +88,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptR import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.Default; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -2423,6 +2424,35 @@ public class TestFairScheduler extends FairSchedulerTestBase { assertEquals(2, defaultQueue.getRunnableAppSchedulables().size()); } + @Test + public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured() + throws IOException { + // This test verifies if default rule in queue placement policy + // initializes properly when policy is not configured and + // undeclared pools is not allowed. + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); + + // Create an alloc file with no queue placement policy + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + List rules = scheduler.allocConf.placementPolicy + .getRules(); + + for (QueuePlacementRule rule : rules) { + if (rule instanceof Default) { + Default defaultRule = (Default) rule; + assertNotNull(defaultRule.defaultQueueName); + } + } + } + @SuppressWarnings("resource") @Test public void testBlacklistNodes() throws Exception { From fc1e525da0527c4eb2a2cd3c95d094b12d316a7d Mon Sep 17 00:00:00 2001 From: Xuan Gong Date: Wed, 28 May 2014 17:10:36 +0000 Subject: [PATCH 18/34] HADOOP-10625. Trim configuration names when putting/getting them to properties git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598072 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/conf/Configuration.java | 16 +++++++++++----- .../apache/hadoop/conf/TestConfiguration.java | 10 +++++++++- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index aa93ffc58f9..c4aaeeb391c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -334,6 +334,9 @@ Trunk (Unreleased) HADOOP-10586. KeyShell doesn't allow setting Options via CLI. (clamb via tucu) + HADOOP-10625. Trim configuration names when putting/getting them + to properties. (Wangda Tan via xgong) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index af345c27cc7..2298d41722e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -566,6 +566,9 @@ public class Configuration implements Iterable>, */ private String[] handleDeprecation(DeprecationContext deprecations, String name) { + if (null != name) { + name = name.trim(); + } ArrayList names = new ArrayList(); if (isDeprecated(name)) { DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name); @@ -843,12 +846,12 @@ public class Configuration implements Iterable>, /** * Get the value of the name property, null if * no such property exists. If the key is deprecated, it returns the value of - * the first key which replaces the deprecated key and is not null + * the first key which replaces the deprecated key and is not null. * * Values are processed for variable expansion * before being returned. * - * @param name the property name. + * @param name the property name, will be trimmed before get value. * @return the value of the name or its replacing property, * or null if no such property exists. */ @@ -952,7 +955,8 @@ public class Configuration implements Iterable>, /** * Set the value of the name property. If * name is deprecated or there is a deprecated name associated to it, - * it sets the value to both names. + * it sets the value to both names. Name will be trimmed before put into + * configuration. * * @param name property name. * @param value property value. @@ -964,7 +968,8 @@ public class Configuration implements Iterable>, /** * Set the value of the name property. If * name is deprecated, it also sets the value to - * the keys that replace the deprecated key. + * the keys that replace the deprecated key. Name will be trimmed before put + * into configuration. * * @param name property name. * @param value property value. @@ -979,6 +984,7 @@ public class Configuration implements Iterable>, Preconditions.checkArgument( value != null, "The value of property " + name + " must not be null"); + name = name.trim(); DeprecationContext deprecations = deprecationContext.get(); if (deprecations.getDeprecatedKeyMap().isEmpty()) { getProps(); @@ -1064,7 +1070,7 @@ public class Configuration implements Iterable>, * If no such property exists, * then defaultValue is returned. * - * @param name property name. + * @param name property name, will be trimmed before get value. * @param defaultValue default value. * @return property value, or defaultValue if the property * doesn't exist. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 1ce0b010851..60b86e44426 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -49,7 +49,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; import static org.apache.hadoop.util.PlatformName.IBM_JAVA; -import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.ObjectMapper; public class TestConfiguration extends TestCase { @@ -1003,6 +1003,14 @@ public class TestConfiguration extends TestCase { String resource; } + public void testGetSetTrimmedNames() throws IOException { + Configuration conf = new Configuration(false); + conf.set(" name", "value"); + assertEquals("value", conf.get("name")); + assertEquals("value", conf.get(" name")); + assertEquals("value", conf.getRaw(" name ")); + } + public void testDumpConfiguration () throws IOException { StringWriter outWriter = new StringWriter(); Configuration.dumpConfiguration(conf, outWriter); From e976652a6e325b0c44172405e49e80ae04c12d0a Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 28 May 2014 17:38:25 +0000 Subject: [PATCH 19/34] HDFS-6442. Fix TestEditLogAutoroll and TestStandbyCheckpoints failure caused by port conficts. (Contributed by Zesheng Wu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598078 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/namenode/TestEditLogAutoroll.java | 47 +++++++++++++------ .../namenode/ha/TestStandbyCheckpoints.java | 44 +++++++++++------ 3 files changed, 65 insertions(+), 29 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5814821b0e3..78c1d370f6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -598,6 +598,9 @@ Release 2.5.0 - UNRELEASED HDFS-6227. ShortCircuitCache#unref should purge ShortCircuitReplicas whose streams have been closed by java interrupts. (Colin Patrick McCabe via jing9) + HDFS-6442. Fix TestEditLogAutoroll and TestStandbyCheckpoints failure + caused by port conficts. (Zesheng Wu via Arpit Agarwal) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java index bea5ee4e567..3a0ae5c5f2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java @@ -17,11 +17,16 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import java.net.BindException; +import java.util.Random; + import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -43,6 +48,9 @@ public class TestEditLogAutoroll { private NameNode nn0; private FileSystem fs; private FSEditLog editLog; + private final Random random = new Random(); + + private static final Log LOG = LogFactory.getLog(TestEditLog.class); @Before public void setUp() throws Exception { @@ -54,24 +62,35 @@ public class TestEditLogAutoroll { conf.setFloat(DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD, 0.5f); conf.setInt(DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS, 100); - MiniDFSNNTopology topology = new MiniDFSNNTopology() - .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10061)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10062))); + int retryCount = 0; + while (true) { + try { + int basePort = 10060 + random.nextInt(100) * 2; + MiniDFSNNTopology topology = new MiniDFSNNTopology() + .addNameservice(new MiniDFSNNTopology.NSConf("ns1") + .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(basePort)) + .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(basePort + 1))); - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(topology) - .numDataNodes(0) - .build(); - cluster.waitActive(); + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(topology) + .numDataNodes(0) + .build(); + cluster.waitActive(); - nn0 = cluster.getNameNode(0); - fs = HATestUtil.configureFailoverFs(cluster, conf); + nn0 = cluster.getNameNode(0); + fs = HATestUtil.configureFailoverFs(cluster, conf); - cluster.transitionToActive(0); + cluster.transitionToActive(0); - fs = cluster.getFileSystem(0); - editLog = nn0.getNamesystem().getEditLog(); + fs = cluster.getFileSystem(0); + editLog = nn0.getNamesystem().getEditLog(); + ++retryCount; + break; + } catch (BindException e) { + LOG.info("Set up MiniDFSCluster failed due to port conflicts, retry " + + retryCount + " times"); + } + } } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java index 2b5e89af8b8..fd579026e78 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java @@ -25,12 +25,14 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.net.BindException; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.net.URI; import java.net.URL; import java.util.List; +import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -73,6 +75,7 @@ public class TestStandbyCheckpoints { protected MiniDFSCluster cluster; protected NameNode nn0, nn1; protected FileSystem fs; + private final Random random = new Random(); protected File tmpOivImgDir; private static final Log LOG = LogFactory.getLog(TestStandbyCheckpoints.class); @@ -87,22 +90,33 @@ public class TestStandbyCheckpoints { conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY, 0); - MiniDFSNNTopology topology = new MiniDFSNNTopology() - .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10061)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10062))); - - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(topology) - .numDataNodes(0) - .build(); - cluster.waitActive(); - - nn0 = cluster.getNameNode(0); - nn1 = cluster.getNameNode(1); - fs = HATestUtil.configureFailoverFs(cluster, conf); + int retryCount = 0; + while (true) { + try { + int basePort = 10060 + random.nextInt(100) * 2; + MiniDFSNNTopology topology = new MiniDFSNNTopology() + .addNameservice(new MiniDFSNNTopology.NSConf("ns1") + .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(basePort)) + .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(basePort + 1))); - cluster.transitionToActive(0); + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(topology) + .numDataNodes(0) + .build(); + cluster.waitActive(); + + nn0 = cluster.getNameNode(0); + nn1 = cluster.getNameNode(1); + fs = HATestUtil.configureFailoverFs(cluster, conf); + + cluster.transitionToActive(0); + ++retryCount; + break; + } catch (BindException e) { + LOG.info("Set up MiniDFSCluster failed due to port conflicts, retry " + + retryCount + " times"); + } + } } protected Configuration setupCommonConfig() { From cfd8647d0f20c08761f908be1f5b718c1c372498 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Wed, 28 May 2014 17:44:14 +0000 Subject: [PATCH 20/34] HDFS-6448. BlockReaderLocalLegacy should set socket timeout based on conf.socketTimeout (liangxie via cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598079 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 78c1d370f6e..8e89d514777 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -601,6 +601,9 @@ Release 2.5.0 - UNRELEASED HDFS-6442. Fix TestEditLogAutoroll and TestStandbyCheckpoints failure caused by port conficts. (Zesheng Wu via Arpit Agarwal) + HDFS-6448. BlockReaderLocalLegacy should set socket timeout based on + conf.socketTimeout (liangxie via cmccabe) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java index ddd25366e02..c68e548099b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java @@ -187,7 +187,7 @@ class BlockReaderLocalLegacy implements BlockReader { userGroupInformation = UserGroupInformation.getCurrentUser(); } pathinfo = getBlockPathInfo(userGroupInformation, blk, node, - configuration, conf.hdfsTimeout, token, + configuration, conf.socketTimeout, token, conf.connectToDnViaHostname); } From 001078e0677e39b962ca1da81fc34d7ac9a7e65c Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Wed, 28 May 2014 18:09:04 +0000 Subject: [PATCH 21/34] YARN-2107. Refactored timeline classes into o.a.h.y.s.timeline package. Contributed by Vinod Kumar Vavilapalli. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598094 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../src/main/resources/yarn-default.xml | 2 +- .../ApplicationHistoryServer.java | 10 +++++----- .../webapp/AHSWebApp.java | 7 ++++--- .../timeline/EntityIdentifier.java | 2 +- .../timeline/GenericObjectMapper.java | 2 +- .../timeline/LeveldbTimelineStore.java | 6 +++--- .../timeline/MemoryTimelineStore.java | 2 +- .../timeline/NameValuePair.java | 2 +- .../timeline/TimelineReader.java | 2 +- .../timeline/TimelineStore.java | 2 +- .../timeline/TimelineWriter.java | 2 +- .../timeline/package-info.java | 2 +- .../timeline/security/TimelineACLsManager.java | 6 +++--- .../security/TimelineAuthenticationFilter.java | 2 +- .../TimelineAuthenticationFilterInitializer.java | 2 +- .../TimelineClientAuthenticationService.java | 2 +- ...imelineDelegationTokenSecretManagerService.java | 2 +- .../webapp/TimelineWebServices.java | 14 +++++++------- .../timeline/TestGenericObjectMapper.java | 4 ++-- .../timeline/TestLeveldbTimelineStore.java | 6 ++++-- .../timeline/TestMemoryTimelineStore.java | 4 +++- .../timeline/TimelineStoreTestUtils.java | 6 ++++-- .../timeline/security/TestTimelineACLsManager.java | 5 +++-- .../webapp/TestTimelineWebServices.java | 9 +++++---- .../apache/hadoop/yarn/server/MiniYARNCluster.java | 5 +++-- .../src/site/apt/TimelineServer.apt.vm | 2 +- 27 files changed, 63 insertions(+), 50 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/EntityIdentifier.java (97%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/GenericObjectMapper.java (98%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/LeveldbTimelineStore.java (99%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/MemoryTimelineStore.java (99%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/NameValuePair.java (95%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/TimelineReader.java (98%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/TimelineStore.java (95%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/TimelineWriter.java (95%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/package-info.java (92%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/security/TimelineACLsManager.java (92%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/security/TimelineAuthenticationFilter.java (95%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/security/TimelineAuthenticationFilterInitializer.java (98%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/security/TimelineClientAuthenticationService.java (99%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/security/TimelineDelegationTokenSecretManagerService.java (98%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => timeline}/webapp/TimelineWebServices.java (96%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/TestGenericObjectMapper.java (95%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/TestLeveldbTimelineStore.java (97%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/TestMemoryTimelineStore.java (92%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/TimelineStoreTestUtils.java (99%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => }/timeline/security/TestTimelineACLsManager.java (94%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/{applicationhistoryservice => timeline}/webapp/TestTimelineWebServices.java (98%) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 81ff5bc7db8..32ee6179708 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -111,6 +111,9 @@ Release 2.5.0 - UNRELEASED YARN-2071. Modified levelDB store permissions to be readable only by the server user. (Zhijie Shen via vinodkv) + YARN-2107. Refactored timeline classes into o.a.h.y.s.timeline package. (Vinod + Kumar Vavilapalli via zjshen) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a528095cfbc..08d13366848 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1143,7 +1143,7 @@ Store class name for timeline store. yarn.timeline-service.store-class - org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore + org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 5e1277f75fc..dfd8c29651c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -38,12 +38,12 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineAuthenticationFilterInitializer; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; +import org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; +import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index 17b1e6286d0..9901eeb4254 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -22,9 +22,10 @@ import static org.apache.hadoop.yarn.util.StringHelper.pajoin; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService; +import org.apache.hadoop.yarn.server.timeline.webapp.TimelineWebServices; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/EntityIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityIdentifier.java similarity index 97% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/EntityIdentifier.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityIdentifier.java index 4b202d88a81..60a9aa74b61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/EntityIdentifier.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityIdentifier.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/GenericObjectMapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/GenericObjectMapper.java similarity index 98% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/GenericObjectMapper.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/GenericObjectMapper.java index b1846a35758..d235ff7536d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/GenericObjectMapper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/GenericObjectMapper.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; import java.io.IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java similarity index 99% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java index 51bffda6274..94957009fba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.readReverseOrderedLong; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; +import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.readReverseOrderedLong; +import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong; import java.io.ByteArrayOutputStream; import java.io.File; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java similarity index 99% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java index 06f3d607a00..b94711c3c61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/MemoryTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; import java.util.ArrayList; import java.util.Collection; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/NameValuePair.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/NameValuePair.java similarity index 95% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/NameValuePair.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/NameValuePair.java index d8dabd2aa54..1f17324bcbd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/NameValuePair.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/NameValuePair.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java similarity index 98% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineReader.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java index 9ae9954dec0..23bca3460e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; import java.io.IOException; import java.util.Collection; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineStore.java similarity index 95% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineStore.java index fc02873bebc..3f6f4abbf1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineStore.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java similarity index 95% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineWriter.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java index 8f28d8272b4..a3e5aeb80b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/package-info.java similarity index 92% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/package-info.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/package-info.java index 970e868e5cb..bf0fe79c319 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/package-info.java @@ -16,5 +16,5 @@ * limitations under the License. */ @InterfaceAudience.Private -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java similarity index 92% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java index 8009b39c94f..848ad0be243 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; +package org.apache.hadoop.yarn.server.timeline.security; import java.io.IOException; import java.util.Set; @@ -29,8 +29,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AdminACLsManager; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore.SystemFilter; +import org.apache.hadoop.yarn.server.timeline.EntityIdentifier; +import org.apache.hadoop.yarn.server.timeline.TimelineStore.SystemFilter; import com.google.common.annotations.VisibleForTesting; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java similarity index 95% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java index 53ef1ed3e84..e6690a6d96a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; +package org.apache.hadoop.yarn.server.timeline.security; import java.util.Properties; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java similarity index 98% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java index e3c303233bb..53b27fbf4bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; +package org.apache.hadoop.yarn.server.timeline.security; import java.io.FileReader; import java.io.IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineClientAuthenticationService.java similarity index 99% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineClientAuthenticationService.java index f11633d9697..9581e82c9ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineClientAuthenticationService.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; +package org.apache.hadoop.yarn.server.timeline.security; import java.io.IOException; import java.io.Writer; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java similarity index 98% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java index 2808dac60da..15395136a58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; +package org.apache.hadoop.yarn.server.timeline.security; import java.io.ByteArrayInputStream; import java.io.DataInputStream; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java similarity index 96% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java index 5d749fa0906..8c7762dad1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; +package org.apache.hadoop.yarn.server.timeline.webapp; import static org.apache.hadoop.yarn.util.StringHelper.CSV_JOINER; @@ -59,12 +59,12 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.NameValuePair; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineReader.Field; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.server.timeline.EntityIdentifier; +import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper; +import org.apache.hadoop.yarn.server.timeline.NameValuePair; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; +import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestGenericObjectMapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java similarity index 95% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestGenericObjectMapper.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java index d684a272b06..ab0923173b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestGenericObjectMapper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java @@ -15,12 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.WritableComparator; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper; +import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper; import org.junit.Test; import java.io.IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java similarity index 97% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java index 59449f82dc1..2adfeaf8dba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; +import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -40,6 +40,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore; +import org.apache.hadoop.yarn.server.timeline.NameValuePair; import org.iq80.leveldb.DBIterator; import org.junit.After; import org.junit.Before; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestMemoryTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java similarity index 92% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestMemoryTimelineStore.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java index 415de536793..1953442f078 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestMemoryTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java @@ -16,9 +16,11 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java similarity index 99% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStoreTestUtils.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java index d76053653ed..e8a6d83050c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStoreTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +package org.apache.hadoop.yarn.server.timeline; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -41,7 +41,9 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineReader.Field; +import org.apache.hadoop.yarn.server.timeline.NameValuePair; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; public class TimelineStoreTestUtils { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java similarity index 94% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java index 39102b43bad..5825e7e0c83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TestTimelineACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java @@ -16,14 +16,15 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; +package org.apache.hadoop.yarn.server.timeline.security; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; import org.junit.Assert; import org.junit.Test; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java similarity index 98% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java index 7e3e4099403..24cc5ca3c63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; +package org.apache.hadoop.yarn.server.timeline.webapp; import static org.junit.Assert.assertEquals; @@ -41,9 +41,10 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AdminACLsManager; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TestMemoryTimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.server.timeline.TestMemoryTimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.server.timeline.webapp.TimelineWebServices.AboutInfo; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.junit.Assert; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 93d0a125f94..b5b7a43faac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -25,6 +25,7 @@ import java.net.UnknownHostException; import java.util.Collection; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -56,8 +57,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.MemoryTimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; @@ -70,6 +69,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptE import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent; +import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm index 3c1169bae80..58f3c627a94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm @@ -152,7 +152,7 @@ YARN Timeline Server Store class name for timeline store. yarn.timeline-service.store-class - org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore + org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore From 4bb4de93d67873b47fd90c61396f52315165c7bf Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Wed, 28 May 2014 19:37:52 +0000 Subject: [PATCH 22/34] MAPREDUCE-5862. Line records longer than 2x split size aren't handled correctly. Contributed by bc Wong git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598111 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../hadoop-mapreduce-client-core/pom.xml | 9 ++ .../hadoop/mapred/LineRecordReader.java | 5 +- .../mapreduce/lib/input/LineRecordReader.java | 5 +- .../hadoop/mapred/TestLineRecordReader.java | 91 +++++++++++++++++ .../lib/input/TestLineRecordReader.java | 92 ++++++++++++++++++ .../recordSpanningMultipleSplits.txt | 4 + .../recordSpanningMultipleSplits.txt.bz2 | Bin 0 -> 99 bytes 8 files changed, 203 insertions(+), 6 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/recordSpanningMultipleSplits.txt create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/recordSpanningMultipleSplits.txt.bz2 diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 82e90984420..5491d1cd5e1 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -241,6 +241,9 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5309. 2.0.4 JobHistoryParser can't parse certain failed job history files generated by 2.0.3 history server (Rushabh S Shah via jlowe) + MAPREDUCE-5862. Line records longer than 2x split size aren't handled + correctly (bc Wong via jlowe) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml index 8b0c4887e84..3ca6dc75e13 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml @@ -85,6 +85,15 @@ + + org.apache.rat + apache-rat-plugin + + + src/test/resources/recordSpanningMultipleSplits.txt + + + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java index 8a1624cd0e6..8b26fbd16d8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java @@ -184,7 +184,7 @@ public class LineRecordReader implements RecordReader { private int maxBytesToConsume(long pos) { return isCompressedInput() ? Integer.MAX_VALUE - : (int) Math.min(Integer.MAX_VALUE, end - pos); + : (int) Math.max(Math.min(Integer.MAX_VALUE, end - pos), maxLineLength); } private long getFilePosition() throws IOException { @@ -206,8 +206,7 @@ public class LineRecordReader implements RecordReader { while (getFilePosition() <= end || in.needAdditionalRecordAfterSplit()) { key.set(pos); - int newSize = in.readLine(value, maxLineLength, - Math.max(maxBytesToConsume(pos), maxLineLength)); + int newSize = in.readLine(value, maxLineLength, maxBytesToConsume(pos)); if (newSize == 0) { return false; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java index 0c7635e4f30..6d12d82d177 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java @@ -121,7 +121,7 @@ public class LineRecordReader extends RecordReader { private int maxBytesToConsume(long pos) { return isCompressedInput ? Integer.MAX_VALUE - : (int) Math.min(Integer.MAX_VALUE, end - pos); + : (int) Math.max(Math.min(Integer.MAX_VALUE, end - pos), maxLineLength); } private long getFilePosition() throws IOException { @@ -146,8 +146,7 @@ public class LineRecordReader extends RecordReader { // We always read one extra line, which lies outside the upper // split limit i.e. (end - 1) while (getFilePosition() <= end || in.needAdditionalRecordAfterSplit()) { - newSize = in.readLine(value, maxLineLength, - Math.max(maxBytesToConsume(pos), maxLineLength)); + newSize = in.readLine(value, maxLineLength, maxBytesToConsume(pos)); pos += newSize; if (newSize < maxLineLength) { break; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java index b8df069fccf..ee066e218f1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java @@ -23,9 +23,12 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.net.URL; +import java.util.ArrayList; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; @@ -97,4 +100,92 @@ public class TestLineRecordReader { // character is a linefeed testSplitRecords("blockEndingInCRThenLF.txt.bz2", 136498); } + + // Use the LineRecordReader to read records from the file + public ArrayList readRecords(URL testFileUrl, int splitSize) + throws IOException { + + // Set up context + File testFile = new File(testFileUrl.getFile()); + long testFileSize = testFile.length(); + Path testFilePath = new Path(testFile.getAbsolutePath()); + Configuration conf = new Configuration(); + conf.setInt("io.file.buffer.size", 1); + + // Gather the records returned by the record reader + ArrayList records = new ArrayList(); + + long offset = 0; + LongWritable key = new LongWritable(); + Text value = new Text(); + while (offset < testFileSize) { + FileSplit split = + new FileSplit(testFilePath, offset, splitSize, (String[]) null); + LineRecordReader reader = new LineRecordReader(conf, split); + + while (reader.next(key, value)) { + records.add(value.toString()); + } + offset += splitSize; + } + return records; + } + + // Gather the records by just splitting on new lines + public String[] readRecordsDirectly(URL testFileUrl, boolean bzip) + throws IOException { + int MAX_DATA_SIZE = 1024 * 1024; + byte[] data = new byte[MAX_DATA_SIZE]; + FileInputStream fis = new FileInputStream(testFileUrl.getFile()); + int count; + if (bzip) { + BZip2CompressorInputStream bzIn = new BZip2CompressorInputStream(fis); + count = bzIn.read(data); + bzIn.close(); + } else { + count = fis.read(data); + } + fis.close(); + assertTrue("Test file data too big for buffer", count < data.length); + return new String(data, 0, count, "UTF-8").split("\n"); + } + + public void checkRecordSpanningMultipleSplits(String testFile, + int splitSize, + boolean bzip) + throws IOException { + URL testFileUrl = getClass().getClassLoader().getResource(testFile); + ArrayList records = readRecords(testFileUrl, splitSize); + String[] actuals = readRecordsDirectly(testFileUrl, bzip); + + assertEquals("Wrong number of records", actuals.length, records.size()); + + boolean hasLargeRecord = false; + for (int i = 0; i < actuals.length; ++i) { + assertEquals(actuals[i], records.get(i)); + if (actuals[i].length() > 2 * splitSize) { + hasLargeRecord = true; + } + } + + assertTrue("Invalid test data. Doesn't have a large enough record", + hasLargeRecord); + } + + @Test + public void testRecordSpanningMultipleSplits() + throws IOException { + checkRecordSpanningMultipleSplits("recordSpanningMultipleSplits.txt", + 10, false); + } + + @Test + public void testRecordSpanningMultipleSplitsCompressed() + throws IOException { + // The file is generated with bz2 block size of 100k. The split size + // needs to be larger than that for the CompressedSplitLineReader to + // work. + checkRecordSpanningMultipleSplits("recordSpanningMultipleSplits.txt.bz2", + 200 * 1000, true); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java index 26c839614ee..46091850ead 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java @@ -23,9 +23,12 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.net.URL; +import java.util.ArrayList; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -101,4 +104,93 @@ public class TestLineRecordReader { // character is a linefeed testSplitRecords("blockEndingInCRThenLF.txt.bz2", 136498); } + + // Use the LineRecordReader to read records from the file + public ArrayList readRecords(URL testFileUrl, int splitSize) + throws IOException { + + // Set up context + File testFile = new File(testFileUrl.getFile()); + long testFileSize = testFile.length(); + Path testFilePath = new Path(testFile.getAbsolutePath()); + Configuration conf = new Configuration(); + conf.setInt("io.file.buffer.size", 1); + TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); + + // Gather the records returned by the record reader + ArrayList records = new ArrayList(); + + long offset = 0; + while (offset < testFileSize) { + FileSplit split = new FileSplit(testFilePath, offset, splitSize, null); + LineRecordReader reader = new LineRecordReader(); + reader.initialize(split, context); + + while (reader.nextKeyValue()) { + records.add(reader.getCurrentValue().toString()); + } + offset += splitSize; + } + return records; + } + + // Gather the records by just splitting on new lines + public String[] readRecordsDirectly(URL testFileUrl, boolean bzip) + throws IOException { + int MAX_DATA_SIZE = 1024 * 1024; + byte[] data = new byte[MAX_DATA_SIZE]; + FileInputStream fis = new FileInputStream(testFileUrl.getFile()); + int count; + if (bzip) { + BZip2CompressorInputStream bzIn = new BZip2CompressorInputStream(fis); + count = bzIn.read(data); + bzIn.close(); + } else { + count = fis.read(data); + } + fis.close(); + assertTrue("Test file data too big for buffer", count < data.length); + return new String(data, 0, count, "UTF-8").split("\n"); + } + + public void checkRecordSpanningMultipleSplits(String testFile, + int splitSize, + boolean bzip) + throws IOException { + URL testFileUrl = getClass().getClassLoader().getResource(testFile); + ArrayList records = readRecords(testFileUrl, splitSize); + String[] actuals = readRecordsDirectly(testFileUrl, bzip); + + assertEquals("Wrong number of records", actuals.length, records.size()); + + boolean hasLargeRecord = false; + for (int i = 0; i < actuals.length; ++i) { + assertEquals(actuals[i], records.get(i)); + if (actuals[i].length() > 2 * splitSize) { + hasLargeRecord = true; + } + } + + assertTrue("Invalid test data. Doesn't have a large enough record", + hasLargeRecord); + } + + @Test + public void testRecordSpanningMultipleSplits() + throws IOException { + checkRecordSpanningMultipleSplits("recordSpanningMultipleSplits.txt", + 10, + false); + } + + @Test + public void testRecordSpanningMultipleSplitsCompressed() + throws IOException { + // The file is generated with bz2 block size of 100k. The split size + // needs to be larger than that for the CompressedSplitLineReader to + // work. + checkRecordSpanningMultipleSplits("recordSpanningMultipleSplits.txt.bz2", + 200 * 1000, + true); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/recordSpanningMultipleSplits.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/recordSpanningMultipleSplits.txt new file mode 100644 index 00000000000..86dc5c1c236 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/recordSpanningMultipleSplits.txt @@ -0,0 +1,4 @@ +Use with small split size, +like 32. +And then we give it a really really long line, which will surely span multiple splits, +to see how it handles. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/recordSpanningMultipleSplits.txt.bz2 b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/recordSpanningMultipleSplits.txt.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..bcd13f5a87b126c54dfa940c3dfd00768131b06a GIT binary patch literal 99 zcmV-p0G$6qT4*sbL0KkKS#Z;6{{R35U4(%UNCI#W2mlgd+>itS00iG> z)EiYjAn_m?G-Z7#)`yxrC>nlA2tYMK00 Date: Wed, 28 May 2014 21:24:49 +0000 Subject: [PATCH 23/34] HDFS-6453. Use Time#monotonicNow to avoid system clock reset. Contributed by Liang Xie. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598144 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/datanode/DataNode.java | 2 +- .../datanode/fsdataset/impl/FsVolumeList.java | 21 ++++++++++--------- .../namenode/FSImageFormatProtobuf.java | 5 +++-- .../hdfs/server/namenode/TransferFsImage.java | 2 +- .../hdfs/server/datanode/TestDiskError.java | 3 ++- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8e89d514777..1fde8757897 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -604,6 +604,9 @@ Release 2.5.0 - UNRELEASED HDFS-6448. BlockReaderLocalLegacy should set socket timeout based on conf.socketTimeout (liangxie via cmccabe) + HDFS-6453. Use Time#monotonicNow to avoid system clock reset. + (Liang Xie via wang) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 84c97bf63c0..bd0cb337d90 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -2581,7 +2581,7 @@ public class DataNode extends Configured return; } synchronized(checkDiskErrorMutex) { - lastDiskErrorCheck = System.currentTimeMillis(); + lastDiskErrorCheck = Time.monotonicNow(); } } try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 9563dcc9870..76b3ee8ba9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -23,6 +23,7 @@ import java.util.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy; +import org.apache.hadoop.util.Time; import org.apache.hadoop.util.DiskChecker.DiskErrorException; class FsVolumeList { @@ -97,7 +98,7 @@ class FsVolumeList { } void getAllVolumesMap(final String bpid, final ReplicaMap volumeMap) throws IOException { - long totalStartTime = System.currentTimeMillis(); + long totalStartTime = Time.monotonicNow(); final List exceptions = Collections.synchronizedList( new ArrayList()); List replicaAddingThreads = new ArrayList(); @@ -107,9 +108,9 @@ class FsVolumeList { try { FsDatasetImpl.LOG.info("Adding replicas to map for block pool " + bpid + " on volume " + v + "..."); - long startTime = System.currentTimeMillis(); + long startTime = Time.monotonicNow(); v.getVolumeMap(bpid, volumeMap); - long timeTaken = System.currentTimeMillis() - startTime; + long timeTaken = Time.monotonicNow() - startTime; FsDatasetImpl.LOG.info("Time to add replicas to map for block pool" + " " + bpid + " on volume " + v + ": " + timeTaken + "ms"); } catch (IOException ioe) { @@ -132,7 +133,7 @@ class FsVolumeList { if (!exceptions.isEmpty()) { throw exceptions.get(0); } - long totalTimeTaken = System.currentTimeMillis() - totalStartTime; + long totalTimeTaken = Time.monotonicNow() - totalStartTime; FsDatasetImpl.LOG.info("Total time to add all replicas to map: " + totalTimeTaken + "ms"); } @@ -141,9 +142,9 @@ class FsVolumeList { throws IOException { FsDatasetImpl.LOG.info("Adding replicas to map for block pool " + bpid + " on volume " + volume + "..."); - long startTime = System.currentTimeMillis(); + long startTime = Time.monotonicNow(); volume.getVolumeMap(bpid, volumeMap); - long timeTaken = System.currentTimeMillis() - startTime; + long timeTaken = Time.monotonicNow() - startTime; FsDatasetImpl.LOG.info("Time to add replicas to map for block pool " + bpid + " on volume " + volume + ": " + timeTaken + "ms"); } @@ -195,7 +196,7 @@ class FsVolumeList { void addBlockPool(final String bpid, final Configuration conf) throws IOException { - long totalStartTime = System.currentTimeMillis(); + long totalStartTime = Time.monotonicNow(); final List exceptions = Collections.synchronizedList( new ArrayList()); @@ -206,9 +207,9 @@ class FsVolumeList { try { FsDatasetImpl.LOG.info("Scanning block pool " + bpid + " on volume " + v + "..."); - long startTime = System.currentTimeMillis(); + long startTime = Time.monotonicNow(); v.addBlockPool(bpid, conf); - long timeTaken = System.currentTimeMillis() - startTime; + long timeTaken = Time.monotonicNow() - startTime; FsDatasetImpl.LOG.info("Time taken to scan block pool " + bpid + " on " + v + ": " + timeTaken + "ms"); } catch (IOException ioe) { @@ -232,7 +233,7 @@ class FsVolumeList { throw exceptions.get(0); } - long totalTimeTaken = System.currentTimeMillis() - totalStartTime; + long totalTimeTaken = Time.monotonicNow() - totalStartTime; FsDatasetImpl.LOG.info("Total time to scan all replicas for block pool " + bpid + ": " + totalTimeTaken + "ms"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java index 5075da9e6f2..a3fb062452e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hdfs.util.MD5FileUtils; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressorStream; +import org.apache.hadoop.util.Time; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -160,13 +161,13 @@ public final class FSImageFormatProtobuf { } void load(File file) throws IOException { - long start = System.currentTimeMillis(); + long start = Time.monotonicNow(); imgDigest = MD5FileUtils.computeMd5ForFile(file); RandomAccessFile raFile = new RandomAccessFile(file, "r"); FileInputStream fin = new FileInputStream(file); try { loadInternal(raFile, fin); - long end = System.currentTimeMillis(); + long end = Time.monotonicNow(); LOG.info("Loaded FSImage in " + (end - start) / 1000 + " seconds."); } finally { fin.close(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java index ca42d075d4b..242c7d75461 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java @@ -159,7 +159,7 @@ public class TransferFsImage { } } - final long milliTime = System.currentTimeMillis(); + final long milliTime = Time.monotonicNow(); String tmpFileName = NNStorage.getTemporaryEditsFileName( log.getStartTxId(), log.getEndTxId(), milliTime); List tmpFiles = dstStorage.getFiles(NameNodeDirType.EDITS, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java index 681ed324e67..f75bf2465ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.util.DataChecksum; +import org.apache.hadoop.util.Time; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -215,6 +216,6 @@ public class TestDiskError { dataNode.checkDiskError(); Thread.sleep(dataNode.checkDiskErrorInterval); long lastDiskErrorCheck = dataNode.getLastDiskErrorCheck(); - assertTrue("Disk Error check is not performed within " + dataNode.checkDiskErrorInterval + " ms", ((System.currentTimeMillis()-lastDiskErrorCheck) < (dataNode.checkDiskErrorInterval + slackTime))); + assertTrue("Disk Error check is not performed within " + dataNode.checkDiskErrorInterval + " ms", ((Time.monotonicNow()-lastDiskErrorCheck) < (dataNode.checkDiskErrorInterval + slackTime))); } } From 342da5b4d3d6f8ae4c9e6c7ece642f7378a6e87c Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Thu, 29 May 2014 04:01:24 +0000 Subject: [PATCH 24/34] YARN-596. Use scheduling policies throughout the queue hierarchy to decide which containers to preempt (Wei Yan via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598197 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/AppSchedulable.java | 45 ++++- .../scheduler/fair/FSLeafQueue.java | 32 +++- .../scheduler/fair/FSParentQueue.java | 28 +++ .../scheduler/fair/FSQueue.java | 13 ++ .../scheduler/fair/FSSchedulerApp.java | 19 ++ .../scheduler/fair/FairScheduler.java | 111 +++++------ .../scheduler/fair/Schedulable.java | 6 + .../scheduler/fair/SchedulingPolicy.java | 10 + .../DominantResourceFairnessPolicy.java | 5 + .../fair/policies/FairSharePolicy.java | 5 + .../scheduler/fair/policies/FifoPolicy.java | 8 +- .../scheduler/fair/FakeSchedulable.java | 6 + .../scheduler/fair/TestFairScheduler.java | 175 +++++++++--------- .../fair/TestFairSchedulerPreemption.java | 5 +- 15 files changed, 309 insertions(+), 162 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 32ee6179708..585545b2333 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -114,6 +114,9 @@ Release 2.5.0 - UNRELEASED YARN-2107. Refactored timeline classes into o.a.h.y.s.timeline package. (Vinod Kumar Vavilapalli via zjshen) + YARN-596. Use scheduling policies throughout the queue hierarchy to decide + which containers to preempt (Wei Yan via Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java index 9ed5179270a..4dc0bf4ceb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java @@ -18,8 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +import java.io.Serializable; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -31,8 +33,6 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; @@ -58,6 +58,8 @@ public class AppSchedulable extends Schedulable { private Priority priority; private ResourceWeights resourceWeights; + private RMContainerComparator comparator = new RMContainerComparator(); + public AppSchedulable(FairScheduler scheduler, FSSchedulerApp app, FSLeafQueue queue) { this.scheduler = scheduler; this.app = app; @@ -111,7 +113,10 @@ public class AppSchedulable extends Schedulable { @Override public Resource getResourceUsage() { - return app.getCurrentConsumption(); + // Here the getPreemptedResources() always return zero, except in + // a preemption round + return Resources.subtract(app.getCurrentConsumption(), + app.getPreemptedResources()); } @@ -383,6 +388,27 @@ public class AppSchedulable extends Schedulable { return assignContainer(node, false); } + /** + * Preempt a running container according to the priority + */ + @Override + public RMContainer preemptContainer() { + if (LOG.isDebugEnabled()) { + LOG.debug("App " + getName() + " is going to preempt a running " + + "container"); + } + + RMContainer toBePreempted = null; + for (RMContainer container : app.getLiveContainers()) { + if (! app.getPreemptionContainers().contains(container) && + (toBePreempted == null || + comparator.compare(toBePreempted, container) > 0)) { + toBePreempted = container; + } + } + return toBePreempted; + } + /** * Whether this app has containers requests that could be satisfied on the * given node, if the node had full space. @@ -407,4 +433,17 @@ public class AppSchedulable extends Schedulable { Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null, anyRequest.getCapability(), node.getRMNode().getTotalCapability()); } + + static class RMContainerComparator implements Comparator, + Serializable { + @Override + public int compare(RMContainer c1, RMContainer c2) { + int ret = c1.getContainer().getPriority().compareTo( + c2.getContainer().getPriority()); + if (ret == 0) { + return c2.getContainerId().compareTo(c1.getContainerId()); + } + return ret; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index e842a6a3557..fe738da7d46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -33,10 +33,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils; import org.apache.hadoop.yarn.util.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; @Private @Unstable @@ -208,6 +208,36 @@ public class FSLeafQueue extends FSQueue { return assigned; } + @Override + public RMContainer preemptContainer() { + RMContainer toBePreempted = null; + if (LOG.isDebugEnabled()) { + LOG.debug("Queue " + getName() + " is going to preempt a container " + + "from its applications."); + } + + // If this queue is not over its fair share, reject + if (!preemptContainerPreCheck()) { + return toBePreempted; + } + + // Choose the app that is most over fair share + Comparator comparator = policy.getComparator(); + AppSchedulable candidateSched = null; + for (AppSchedulable sched : runnableAppScheds) { + if (candidateSched == null || + comparator.compare(sched, candidateSched) > 0) { + candidateSched = sched; + } + } + + // Preempt from the selected app + if (candidateSched != null) { + toBePreempted = candidateSched.preemptContainer(); + } + return toBePreempted; + } + @Override public List getChildQueues() { return new ArrayList(1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index 427cb864579..48db4149634 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.List; import org.apache.commons.logging.Log; @@ -32,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; @@ -156,6 +158,32 @@ public class FSParentQueue extends FSQueue { return assigned; } + @Override + public RMContainer preemptContainer() { + RMContainer toBePreempted = null; + + // If this queue is not over its fair share, reject + if (!preemptContainerPreCheck()) { + return toBePreempted; + } + + // Find the childQueue which is most over fair share + FSQueue candidateQueue = null; + Comparator comparator = policy.getComparator(); + for (FSQueue queue : childQueues) { + if (candidateQueue == null || + comparator.compare(queue, candidateQueue) > 0) { + candidateQueue = queue; + } + } + + // Let the selected queue choose which of its container to preempt + if (candidateQueue != null) { + toBePreempted = candidateQueue.preemptContainer(); + } + return toBePreempted; + } + @Override public List getChildQueues() { return childQueues; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 1e94046100a..716e1ee6874 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -187,4 +187,17 @@ public abstract class FSQueue extends Schedulable implements Queue { } return true; } + + /** + * Helper method to check if the queue should preempt containers + * + * @return true if check passes (can preempt) or false otherwise + */ + protected boolean preemptContainerPreCheck() { + if (this == scheduler.getQueueManager().getRootQueue()) { + return true; + } + return parent.getPolicy() + .checkIfUsageOverFairShare(getResourceUsage(), getFairShare()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index adabfefaee1..63a29e4b099 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -59,6 +59,8 @@ public class FSSchedulerApp extends SchedulerApplicationAttempt { private AppSchedulable appSchedulable; final Map preemptionMap = new HashMap(); + + private Resource preemptedResources = Resources.createResource(0); public FSSchedulerApp(ApplicationAttemptId applicationAttemptId, String user, FSLeafQueue queue, ActiveUsersManager activeUsersManager, @@ -316,6 +318,7 @@ public class FSSchedulerApp extends SchedulerApplicationAttempt { public void addPreemption(RMContainer container, long time) { assert preemptionMap.get(container) == null; preemptionMap.put(container, time); + Resources.addTo(preemptedResources, container.getAllocatedResource()); } public Long getContainerPreemptionTime(RMContainer container) { @@ -330,4 +333,20 @@ public class FSSchedulerApp extends SchedulerApplicationAttempt { public FSLeafQueue getQueue() { return (FSLeafQueue)super.getQueue(); } + + public Resource getPreemptedResources() { + return preemptedResources; + } + + public void resetPreemptedResources() { + preemptedResources = Resources.createResource(0); + for (RMContainer container : getPreemptionContainers()) { + Resources.addTo(preemptedResources, container.getAllocatedResource()); + } + } + + public void clearPreemptedResources() { + preemptedResources.setMemory(0); + preemptedResources.setVirtualCores(0); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 830f6f75099..6d71ea2fbb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -20,14 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -337,94 +334,78 @@ public class FairScheduler extends } if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, resToPreempt, Resources.none())) { - preemptResources(queueMgr.getLeafQueues(), resToPreempt); + preemptResources(resToPreempt); } } /** - * Preempt a quantity of resources from a list of QueueSchedulables. The - * policy for this is to pick apps from queues that are over their fair share, - * but make sure that no queue is placed below its fair share in the process. - * We further prioritize preemption by choosing containers with lowest - * priority to preempt. + * Preempt a quantity of resources. Each round, we start from the root queue, + * level-by-level, until choosing a candidate application. + * The policy for prioritizing preemption for each queue depends on its + * SchedulingPolicy: (1) fairshare/DRF, choose the ChildSchedulable that is + * most over its fair share; (2) FIFO, choose the childSchedulable that is + * latest launched. + * Inside each application, we further prioritize preemption by choosing + * containers with lowest priority to preempt. + * We make sure that no queue is placed below its fair share in the process. */ - protected void preemptResources(Collection scheds, - Resource toPreempt) { - if (scheds.isEmpty() || Resources.equals(toPreempt, Resources.none())) { + protected void preemptResources(Resource toPreempt) { + if (Resources.equals(toPreempt, Resources.none())) { return; } - Map apps = - new HashMap(); - Map queues = - new HashMap(); - - // Collect running containers from over-scheduled queues - List runningContainers = new ArrayList(); - for (FSLeafQueue sched : scheds) { - if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, - sched.getResourceUsage(), sched.getFairShare())) { - for (AppSchedulable as : sched.getRunnableAppSchedulables()) { - for (RMContainer c : as.getApp().getLiveContainers()) { - runningContainers.add(c); - apps.put(c, as.getApp()); - queues.put(c, sched); - } - } - } - } - - // Sort containers into reverse order of priority - Collections.sort(runningContainers, new Comparator() { - public int compare(RMContainer c1, RMContainer c2) { - int ret = c1.getContainer().getPriority().compareTo( - c2.getContainer().getPriority()); - if (ret == 0) { - return c2.getContainerId().compareTo(c1.getContainerId()); - } - return ret; - } - }); - // Scan down the list of containers we've already warned and kill them // if we need to. Remove any containers from the list that we don't need // or that are no longer running. Iterator warnedIter = warnedContainers.iterator(); - Set preemptedThisRound = new HashSet(); while (warnedIter.hasNext()) { RMContainer container = warnedIter.next(); - if (container.getState() == RMContainerState.RUNNING && + if ((container.getState() == RMContainerState.RUNNING || + container.getState() == RMContainerState.ALLOCATED) && Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, toPreempt, Resources.none())) { - warnOrKillContainer(container, apps.get(container), queues.get(container)); - preemptedThisRound.add(container); + warnOrKillContainer(container); Resources.subtractFrom(toPreempt, container.getContainer().getResource()); } else { warnedIter.remove(); } } - // Scan down the rest of the containers until we've preempted enough, making - // sure we don't preempt too many from any queue - Iterator runningIter = runningContainers.iterator(); - while (runningIter.hasNext() && - Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, - toPreempt, Resources.none())) { - RMContainer container = runningIter.next(); - FSLeafQueue sched = queues.get(container); - if (!preemptedThisRound.contains(container) && - Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, - sched.getResourceUsage(), sched.getFairShare())) { - warnOrKillContainer(container, apps.get(container), sched); - - warnedContainers.add(container); - Resources.subtractFrom(toPreempt, container.getContainer().getResource()); + try { + // Reset preemptedResource for each app + for (FSLeafQueue queue : getQueueManager().getLeafQueues()) { + for (AppSchedulable app : queue.getRunnableAppSchedulables()) { + app.getApp().resetPreemptedResources(); + } + } + + while (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, + toPreempt, Resources.none())) { + RMContainer container = + getQueueManager().getRootQueue().preemptContainer(); + if (container == null) { + break; + } else { + warnOrKillContainer(container); + warnedContainers.add(container); + Resources.subtractFrom( + toPreempt, container.getContainer().getResource()); + } + } + } finally { + // Clear preemptedResources for each app + for (FSLeafQueue queue : getQueueManager().getLeafQueues()) { + for (AppSchedulable app : queue.getRunnableAppSchedulables()) { + app.getApp().clearPreemptedResources(); + } } } } - private void warnOrKillContainer(RMContainer container, FSSchedulerApp app, - FSLeafQueue queue) { + private void warnOrKillContainer(RMContainer container) { + ApplicationAttemptId appAttemptId = container.getApplicationAttemptId(); + FSSchedulerApp app = getSchedulerApp(appAttemptId); + FSLeafQueue queue = app.getQueue(); LOG.info("Preempting container (prio=" + container.getContainer().getPriority() + "res=" + container.getContainer().getResource() + ") from queue " + queue.getName()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java index 92b6d3e71ea..4f8ac1e6374 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.util.resource.Resources; /** @@ -100,6 +101,11 @@ public abstract class Schedulable { */ public abstract Resource assignContainer(FSSchedulerNode node); + /** + * Preempt a container from this Schedulable if possible. + */ + public abstract RMContainer preemptContainer(); + /** Assign a fair share to this Schedulable. */ public void setFairShare(Resource fairShare) { this.fairShare = fairShare; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java index 549b85c380f..1d77a43ce75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java @@ -139,4 +139,14 @@ public abstract class SchedulingPolicy { */ public abstract void computeShares( Collection schedulables, Resource totalResources); + + /** + * Check if the resource usage is over the fair share under this policy + * + * @param usage {@link Resource} the resource usage + * @param fairShare {@link Resource} the fair share + * @return true if check passes (is over) or false otherwise + */ + public abstract boolean checkIfUsageOverFairShare( + Resource usage, Resource fairShare); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java index f5b84177229..4b663d95de8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java @@ -69,6 +69,11 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy { } } + @Override + public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) { + return !Resources.fitsIn(usage, fairShare); + } + @Override public void initialize(Resource clusterCapacity) { comparator.setClusterCapacity(clusterCapacity); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java index fbad1012676..ca7297ff46c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java @@ -119,6 +119,11 @@ public class FairSharePolicy extends SchedulingPolicy { ComputeFairShares.computeShares(schedulables, totalResources, ResourceType.MEMORY); } + @Override + public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) { + return Resources.greaterThan(RESOURCE_CALCULATOR, null, usage, fairShare); + } + @Override public byte getApplicableDepth() { return SchedulingPolicy.DEPTH_ANY; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java index 3451cfea4c5..d9969446811 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies; import java.io.Serializable; import java.util.Collection; import java.util.Comparator; -import java.util.Iterator; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -88,6 +87,13 @@ public class FifoPolicy extends SchedulingPolicy { earliest.setFairShare(Resources.clone(totalResources)); } + @Override + public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) { + throw new UnsupportedOperationException( + "FifoPolicy doesn't support checkIfUsageOverFairshare operation, " + + "as FifoPolicy only works for FSLeafQueue."); + } + @Override public byte getApplicableDepth() { return SchedulingPolicy.DEPTH_LEAF; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java index d0ba0d8e085..dcfc2d3aa2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; @@ -83,6 +84,11 @@ public class FakeSchedulable extends Schedulable { return null; } + @Override + public RMContainer preemptContainer() { + return null; + } + @Override public Resource getDemand() { return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 276020f00c9..bda9564f196 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -1029,13 +1029,13 @@ public class TestFairScheduler extends FairSchedulerTestBase { @Test (timeout = 5000) /** - * Make sure containers are chosen to be preempted in the correct order. Right - * now this means decreasing order of priority. + * Make sure containers are chosen to be preempted in the correct order. */ public void testChoiceOfPreemptedContainers() throws Exception { conf.setLong(FairSchedulerConfiguration.PREEMPTION_INTERVAL, 5000); - conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10000); + conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10000); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE + ".allocation.file", ALLOC_FILE); + conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); MockClock clock = new MockClock(); scheduler.setClock(clock); @@ -1052,7 +1052,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { out.println(""); out.println(".25"); out.println(""); - out.println(""); + out.println(""); out.println(".25"); out.println(""); out.println(""); @@ -1060,133 +1060,132 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.reinitialize(conf, resourceManager.getRMContext()); - // Create four nodes + // Create two nodes RMNode node1 = - MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 2), 1, + MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); RMNode node2 = - MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 2), 2, + MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 2, "127.0.0.2"); NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); scheduler.handle(nodeEvent2); - RMNode node3 = - MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 2), 3, - "127.0.0.3"); - NodeAddedSchedulerEvent nodeEvent3 = new NodeAddedSchedulerEvent(node3); - scheduler.handle(nodeEvent3); - - - // Queue A and B each request three containers + // Queue A and B each request two applications ApplicationAttemptId app1 = - createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 1); + createSchedulingRequest(1 * 1024, 1, "queueA", "user1", 1, 1); + createSchedulingRequestExistingApplication(1 * 1024, 1, 2, app1); ApplicationAttemptId app2 = - createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 2); - ApplicationAttemptId app3 = - createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 3); + createSchedulingRequest(1 * 1024, 1, "queueA", "user1", 1, 3); + createSchedulingRequestExistingApplication(1 * 1024, 1, 4, app2); + ApplicationAttemptId app3 = + createSchedulingRequest(1 * 1024, 1, "queueB", "user1", 1, 1); + createSchedulingRequestExistingApplication(1 * 1024, 1, 2, app3); ApplicationAttemptId app4 = - createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 1); - ApplicationAttemptId app5 = - createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 2); - ApplicationAttemptId app6 = - createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 3); + createSchedulingRequest(1 * 1024, 1, "queueB", "user1", 1, 3); + createSchedulingRequestExistingApplication(1 * 1024, 1, 4, app4); scheduler.update(); + scheduler.getQueueManager().getLeafQueue("queueA", true) + .setPolicy(SchedulingPolicy.parse("fifo")); + scheduler.getQueueManager().getLeafQueue("queueB", true) + .setPolicy(SchedulingPolicy.parse("fair")); + // Sufficient node check-ins to fully schedule containers - for (int i = 0; i < 2; i++) { - NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1); + NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1); + NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2); + for (int i = 0; i < 4; i++) { scheduler.handle(nodeUpdate1); - - NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2); scheduler.handle(nodeUpdate2); - - NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3); - scheduler.handle(nodeUpdate3); } - assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size()); - assertEquals(1, scheduler.getSchedulerApp(app2).getLiveContainers().size()); - assertEquals(1, scheduler.getSchedulerApp(app3).getLiveContainers().size()); - assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size()); - assertEquals(1, scheduler.getSchedulerApp(app5).getLiveContainers().size()); - assertEquals(1, scheduler.getSchedulerApp(app6).getLiveContainers().size()); - - // Now new requests arrive from queues C and D - ApplicationAttemptId app7 = - createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 1); - ApplicationAttemptId app8 = - createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 2); - ApplicationAttemptId app9 = - createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 3); - - ApplicationAttemptId app10 = - createSchedulingRequest(1 * 1024, "queueD", "user1", 1, 1); - ApplicationAttemptId app11 = - createSchedulingRequest(1 * 1024, "queueD", "user1", 1, 2); - ApplicationAttemptId app12 = - createSchedulingRequest(1 * 1024, "queueD", "user1", 1, 3); + assertEquals(2, scheduler.getSchedulerApp(app1).getLiveContainers().size()); + assertEquals(2, scheduler.getSchedulerApp(app2).getLiveContainers().size()); + assertEquals(2, scheduler.getSchedulerApp(app3).getLiveContainers().size()); + assertEquals(2, scheduler.getSchedulerApp(app4).getLiveContainers().size()); + // Now new requests arrive from queueC and default + createSchedulingRequest(1 * 1024, 1, "queueC", "user1", 1, 1); + createSchedulingRequest(1 * 1024, 1, "queueC", "user1", 1, 1); + createSchedulingRequest(1 * 1024, 1, "default", "user1", 1, 1); + createSchedulingRequest(1 * 1024, 1, "default", "user1", 1, 1); scheduler.update(); - // We should be able to claw back one container from A and B each. - // Make sure it is lowest priority container. - scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(), - Resources.createResource(2 * 1024)); - assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size()); - assertEquals(1, scheduler.getSchedulerApp(app2).getLiveContainers().size()); - assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size()); - assertEquals(1, scheduler.getSchedulerApp(app5).getLiveContainers().size()); - - // First verify we are adding containers to preemption list for the application - assertTrue(!Collections.disjoint(scheduler.getSchedulerApp(app3).getLiveContainers(), - scheduler.getSchedulerApp(app3).getPreemptionContainers())); - assertTrue(!Collections.disjoint(scheduler.getSchedulerApp(app6).getLiveContainers(), - scheduler.getSchedulerApp(app6).getPreemptionContainers())); + // We should be able to claw back one container from queueA and queueB each. + scheduler.preemptResources(Resources.createResource(2 * 1024)); + assertEquals(2, scheduler.getSchedulerApp(app1).getLiveContainers().size()); + assertEquals(2, scheduler.getSchedulerApp(app3).getLiveContainers().size()); + + // First verify we are adding containers to preemption list for the app. + // For queueA (fifo), app2 is selected. + // For queueB (fair), app4 is selected. + assertTrue("App2 should have container to be preempted", + !Collections.disjoint( + scheduler.getSchedulerApp(app2).getLiveContainers(), + scheduler.getSchedulerApp(app2).getPreemptionContainers())); + assertTrue("App4 should have container to be preempted", + !Collections.disjoint( + scheduler.getSchedulerApp(app2).getLiveContainers(), + scheduler.getSchedulerApp(app2).getPreemptionContainers())); // Pretend 15 seconds have passed clock.tick(15); // Trigger a kill by insisting we want containers back - scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(), - Resources.createResource(2 * 1024)); + scheduler.preemptResources(Resources.createResource(2 * 1024)); // At this point the containers should have been killed (since we are not simulating AM) - assertEquals(0, scheduler.getSchedulerApp(app6).getLiveContainers().size()); - assertEquals(0, scheduler.getSchedulerApp(app3).getLiveContainers().size()); + assertEquals(1, scheduler.getSchedulerApp(app2).getLiveContainers().size()); + assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size()); + // Inside each app, containers are sorted according to their priorities. + // Containers with priority 4 are preempted for app2 and app4. + Set set = new HashSet(); + for (RMContainer container : + scheduler.getSchedulerApp(app2).getLiveContainers()) { + if (container.getAllocatedPriority().getPriority() == 4) { + set.add(container); + } + } + for (RMContainer container : + scheduler.getSchedulerApp(app4).getLiveContainers()) { + if (container.getAllocatedPriority().getPriority() == 4) { + set.add(container); + } + } + assertTrue("Containers with priority=4 in app2 and app4 should be " + + "preempted.", set.isEmpty()); // Trigger a kill by insisting we want containers back - scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(), - Resources.createResource(2 * 1024)); + scheduler.preemptResources(Resources.createResource(2 * 1024)); // Pretend 15 seconds have passed clock.tick(15); // We should be able to claw back another container from A and B each. - // Make sure it is lowest priority container. - scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(), - Resources.createResource(2 * 1024)); - - assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size()); + // For queueA (fifo), continue preempting from app2. + // For queueB (fair), even app4 has a lowest priority container with p=4, it + // still preempts from app3 as app3 is most over fair share. + scheduler.preemptResources(Resources.createResource(2 * 1024)); + + assertEquals(2, scheduler.getSchedulerApp(app1).getLiveContainers().size()); assertEquals(0, scheduler.getSchedulerApp(app2).getLiveContainers().size()); - assertEquals(0, scheduler.getSchedulerApp(app3).getLiveContainers().size()); + assertEquals(1, scheduler.getSchedulerApp(app3).getLiveContainers().size()); assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size()); - assertEquals(0, scheduler.getSchedulerApp(app5).getLiveContainers().size()); - assertEquals(0, scheduler.getSchedulerApp(app6).getLiveContainers().size()); // Now A and B are below fair share, so preemption shouldn't do anything - scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(), - Resources.createResource(2 * 1024)); - assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size()); - assertEquals(0, scheduler.getSchedulerApp(app2).getLiveContainers().size()); - assertEquals(0, scheduler.getSchedulerApp(app3).getLiveContainers().size()); - assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size()); - assertEquals(0, scheduler.getSchedulerApp(app5).getLiveContainers().size()); - assertEquals(0, scheduler.getSchedulerApp(app6).getLiveContainers().size()); + scheduler.preemptResources(Resources.createResource(2 * 1024)); + assertTrue("App1 should have no container to be preempted", + scheduler.getSchedulerApp(app1).getPreemptionContainers().isEmpty()); + assertTrue("App2 should have no container to be preempted", + scheduler.getSchedulerApp(app2).getPreemptionContainers().isEmpty()); + assertTrue("App3 should have no container to be preempted", + scheduler.getSchedulerApp(app3).getPreemptionContainers().isEmpty()); + assertTrue("App4 should have no container to be preempted", + scheduler.getSchedulerApp(app4).getPreemptionContainers().isEmpty()); } @Test (timeout = 5000) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java index b3ab299ea88..2098e1679b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java @@ -35,10 +35,8 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; -import java.util.Collection; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; public class TestFairSchedulerPreemption extends FairSchedulerTestBase { @@ -51,8 +49,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase { public int lastPreemptMemory = -1; @Override - protected void preemptResources( - Collection scheds, Resource toPreempt) { + protected void preemptResources(Resource toPreempt) { lastPreemptMemory = toPreempt.getMemory(); } From 61c59c07043e5d6d13ef8bec1fc10e6db8317a70 Mon Sep 17 00:00:00 2001 From: Devarajulu K Date: Thu, 29 May 2014 07:05:26 +0000 Subject: [PATCH 25/34] MAPREDUCE-5895. Close streams properly to avoid leakage in TaskLog. Contributed by Kousuke Saruta. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598209 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/mapred/TaskLog.java | 24 +++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 5491d1cd5e1..835ce619240 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -244,6 +244,9 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5862. Line records longer than 2x split size aren't handled correctly (bc Wong via jlowe) + MAPREDUCE-5895. Close streams properly to avoid leakage in TaskLog. + (Kousuke Saruta via devaraj) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java index 48640ab79c4..a86e76a4cea 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java @@ -199,16 +199,18 @@ public class TaskLog { // file first and then rename. File tmpIndexFile = getTmpIndexFile(currentTaskid, isCleanup); - BufferedOutputStream bos = - new BufferedOutputStream( - SecureIOUtils.createForWrite(tmpIndexFile, 0644)); - DataOutputStream dos = new DataOutputStream(bos); - //the format of the index file is - //LOG_DIR: - //STDOUT: - //STDERR: - //SYSLOG: + BufferedOutputStream bos = null; + DataOutputStream dos = null; try{ + bos = new BufferedOutputStream( + SecureIOUtils.createForWrite(tmpIndexFile, 0644)); + dos = new DataOutputStream(bos); + //the format of the index file is + //LOG_DIR: + //STDOUT: + //STDERR: + //SYSLOG: + dos.writeBytes(LogFileDetail.LOCATION + logLocation + "\n" + LogName.STDOUT.toString() + ":"); dos.writeBytes(Long.toString(prevOutLength) + " "); @@ -225,8 +227,10 @@ public class TaskLog { + "\n"); dos.close(); dos = null; + bos.close(); + bos = null; } finally { - IOUtils.cleanup(LOG, dos); + IOUtils.cleanup(LOG, dos, bos); } File indexFile = getIndexFile(currentTaskid, isCleanup); From 806f4d11f2b339b9c935f46d0b92641acefabf77 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 29 May 2014 17:26:50 +0000 Subject: [PATCH 26/34] HADOOP-10602. Documentation has broken "Go Back" hyperlinks. Contributed by Akira AJISAKA. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598337 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-auth/src/site/apt/BuildingIt.apt.vm | 5 ----- .../hadoop-auth/src/site/apt/Configuration.apt.vm | 4 ---- .../hadoop-auth/src/site/apt/Examples.apt.vm | 4 ---- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm | 2 -- .../hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm | 4 ---- .../hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm | 4 ---- .../src/site/apt/CentralizedCacheManagement.apt.vm | 2 -- .../hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm | 2 -- .../hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm | 2 -- hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ViewFs.apt.vm | 2 -- .../src/site/apt/DistributedCacheDeploy.apt.vm | 2 -- .../src/site/apt/EncryptedShuffle.apt.vm | 2 -- .../site/apt/MapReduce_Compatibility_Hadoop1_Hadoop2.apt.vm | 2 -- .../src/site/apt/PluggableShuffleAndPluggableSort.apt.vm | 2 -- .../hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm | 2 -- .../hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm | 2 -- .../hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm | 2 -- .../hadoop-yarn-site/src/site/apt/NodeManager.apt.vm | 2 -- .../hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm | 2 -- .../hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm | 2 -- .../hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm | 2 -- .../hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm | 2 -- .../hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm | 2 -- .../src/site/apt/WritingYarnApplications.apt.vm | 2 -- .../hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm | 2 -- 26 files changed, 3 insertions(+), 61 deletions(-) diff --git a/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm b/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm index a2e015ae0cd..2ca2f0a7ed5 100644 --- a/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm +++ b/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm @@ -18,8 +18,6 @@ Hadoop Auth, Java HTTP SPNEGO ${project.version} - Building It - \[ {{{./index.html}Go Back}} \] - * Requirements * Java 6+ @@ -70,6 +68,3 @@ $ mvn package -Pdocs The generated documentation is available at <<>>. - - \[ {{{./index.html}Go Back}} \] - diff --git a/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm b/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm index 61bd857515e..63773933551 100644 --- a/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm +++ b/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm @@ -20,8 +20,6 @@ Hadoop Auth, Java HTTP SPNEGO ${project.version} - Server Side Configuration - \[ {{{./index.html}Go Back}} \] - * Server Side Configuration Setup The AuthenticationFilter filter is Hadoop Auth's server side component. @@ -241,5 +239,3 @@ Configuration ... +---+ - - \[ {{{./index.html}Go Back}} \] diff --git a/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm b/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm index 7070862d9e3..1b1afd5fdef 100644 --- a/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm +++ b/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm @@ -18,8 +18,6 @@ Hadoop Auth, Java HTTP SPNEGO ${project.version} - Examples - \[ {{{./index.html}Go Back}} \] - * Accessing a Hadoop Auth protected URL Using a browser <> The browser must support HTTP Kerberos SPNEGO. For example, @@ -133,5 +131,3 @@ You are: user[tucu] principal[tucu@LOCALHOST] .... +---+ - - \[ {{{./index.html}Go Back}} \] diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c4aaeeb391c..8fe4012bd57 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -510,6 +510,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10489. UserGroupInformation#getTokens and UserGroupInformation #addToken can lead to ConcurrentModificationException (Robert Kanter via atm) + HADOOP-10489. Documentation has broken "Go Back" hyperlinks. + (Akira AJISAKA via cnauroth) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm index e4d445dcccf..329fd516e30 100644 --- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm +++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm @@ -483,5 +483,3 @@ Content-Type: application/json ... ] +---+ - - \[ {{{./index.html}Go Back}} \] diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm index 42e4656c1dc..878ab1f7747 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm @@ -18,8 +18,6 @@ Hadoop HDFS over HTTP ${project.version} - Server Setup - \[ {{{./index.html}Go Back}} \] - This page explains how to quickly setup HttpFS with Pseudo authentication against a Hadoop cluster with Pseudo authentication. @@ -159,5 +157,3 @@ $ keytool -genkey -alias tomcat -keyalg RSA <<>> scheme. Make sure the JVM is picking up the truststore containing the public key of the SSL certificate if using a self-signed certificate. - - \[ {{{./index.html}Go Back}} \] diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm index 4bd137950fd..c93e20b9d80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm @@ -18,8 +18,6 @@ Hadoop HDFS over HTTP ${project.version} - Using HTTP Tools - \[ {{{./index.html}Go Back}} \] - * Security Out of the box HttpFS supports both pseudo authentication and Kerberos HTTP @@ -87,5 +85,3 @@ $ curl --negotiate -u foo -c ~/.httpfsauth "http://:14000/webhdfs/v +---+ $ curl -b ~/.httpfsauth "http://:14000/webhdfs/v1?op=liststatus" +---+ - - \[ {{{./index.html}Go Back}} \] diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm index bd2653e862b..d3a712294ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm @@ -18,8 +18,6 @@ Centralized Cache Management in HDFS - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=2|toDepth=4} * {Overview} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm index af6132bdef5..56aec0ca288 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm @@ -18,8 +18,6 @@ Extended Attributes in HDFS - \[ {{{../../index.html}Go Back}} \] - %{toc|section=1|fromDepth=2|toDepth=4} * {Overview} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm index 7f8e8211b10..1d164a2d838 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm @@ -19,8 +19,6 @@ HDFS NFS Gateway - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * {Overview} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ViewFs.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ViewFs.apt.vm index ec0f39adabb..4868a20ec6a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ViewFs.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ViewFs.apt.vm @@ -18,8 +18,6 @@ ViewFs Guide - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * {Introduction} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm index 302f8d110af..f4c5eb40939 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm @@ -18,8 +18,6 @@ Hadoop MapReduce Next Generation - Distributed Cache Deploy - \[ {{{./index.html}Go Back}} \] - * Introduction The MapReduce application framework has rudimentary support for deploying a diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm index e05951c1797..e766cbc10ed 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm @@ -18,8 +18,6 @@ Hadoop MapReduce Next Generation - Encrypted Shuffle - \[ {{{./index.html}Go Back}} \] - * {Introduction} The Encrypted Shuffle capability allows encryption of the MapReduce shuffle diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduce_Compatibility_Hadoop1_Hadoop2.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduce_Compatibility_Hadoop1_Hadoop2.apt.vm index bdc2f36a0d6..e0fce633245 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduce_Compatibility_Hadoop1_Hadoop2.apt.vm +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduce_Compatibility_Hadoop1_Hadoop2.apt.vm @@ -18,8 +18,6 @@ Apache Hadoop MapReduce - Migrating from Apache Hadoop 1.x to Apache Hadoop 2.x - \[ {{{../../hadoop-yarn/hadoop-yarn-site/index.html}Go Back}} \] - * {Introduction} This document provides information for users to migrate their Apache Hadoop diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm index c37fc071d87..1b06ca9bfbe 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm @@ -18,8 +18,6 @@ Hadoop MapReduce Next Generation - Pluggable Shuffle and Pluggable Sort - \[ {{{./index.html}Go Back}} \] - * Introduction The pluggable shuffle and pluggable sort capabilities allow replacing the diff --git a/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm b/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm index b772524bb41..399a1f5e8f8 100644 --- a/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm +++ b/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm @@ -18,8 +18,6 @@ Yarn Scheduler Load Simulator (SLS) - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * Overview diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm index 6bd9a7bf6f5..bc3b9eaa5c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm @@ -18,8 +18,6 @@ Hadoop MapReduce Next Generation - Capacity Scheduler - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * {Purpose} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm index 48011cbe50e..54daf2da859 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm @@ -18,8 +18,6 @@ Hadoop MapReduce Next Generation - Fair Scheduler - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * {Purpose} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm index cdf8b055a9f..9ee942fad4d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm @@ -18,8 +18,6 @@ NodeManager Overview. - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0|toDepth=2} * Overview diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm index c848cb4b0ac..36208f4e414 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm @@ -18,8 +18,6 @@ NodeManager REST API's. - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0|toDepth=2} * Overview diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm index 7da3f351b39..8cfdd79b00e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm @@ -18,8 +18,6 @@ ResourceManager High Availability - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * Introduction diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index ac6b4466c27..a8f4f007ad2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -18,8 +18,6 @@ ResourceManager REST API's. - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0|toDepth=2} * Overview diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm index 58f3c627a94..c704d3797f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm @@ -18,8 +18,6 @@ YARN Timeline Server - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0|toDepth=3} * Overview diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm index 68d531e85c5..5300b94b9ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm @@ -18,8 +18,6 @@ Hadoop YARN - Introduction to the web services REST API's. - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * Overview diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm index 698ffea3942..12e4c3ff334 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm @@ -19,8 +19,6 @@ Hadoop MapReduce Next Generation - Writing YARN Applications - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * Purpose diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm index d4312b4086f..018949e717a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm @@ -18,8 +18,6 @@ Yarn Commands - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * Overview From 3c5ce2df946a4a839ed077255ec9ae5864a31d7d Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 29 May 2014 17:34:17 +0000 Subject: [PATCH 27/34] HADOOP-10602. Correct CHANGES.txt. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598339 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 8fe4012bd57..9a182057a89 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -510,7 +510,7 @@ Release 2.5.0 - UNRELEASED HADOOP-10489. UserGroupInformation#getTokens and UserGroupInformation #addToken can lead to ConcurrentModificationException (Robert Kanter via atm) - HADOOP-10489. Documentation has broken "Go Back" hyperlinks. + HADOOP-10602. Documentation has broken "Go Back" hyperlinks. (Akira AJISAKA via cnauroth) Release 2.4.1 - UNRELEASED From cd24e955cf7720cc4ad7687a0341b6e383722b64 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 29 May 2014 19:06:36 +0000 Subject: [PATCH 28/34] YARN-2112. Fixed yarn-common's pom.xml to include jackson dependencies so that both Timeline Server and client can access them. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598373 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 ++++ .../hadoop-yarn/hadoop-yarn-common/pom.xml | 14 +++++++++++++- .../pom.xml | 9 --------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 585545b2333..4b2f77ea616 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -158,6 +158,10 @@ Release 2.5.0 - UNRELEASED YARN-2105. Fix TestFairScheduler after YARN-2012. (Ashwin Shankar via Sandy Ryza) + YARN-2112. Fixed yarn-common's pom.xml to include jackson dependencies so + that both Timeline Server and client can access them. (Zhijie Shen via + vinodkv) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index 9f2e8b5aa8e..aa12b3f55a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -72,7 +72,19 @@ org.codehaus.jackson - jackson-mapper-asl + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc com.google.guava diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index 616c7f0aa00..71605a5bf6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -150,15 +150,6 @@ test - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-mapper-asl - - commons-collections commons-collections From 4a4868e52360d7543909a7981b4606db939230ea Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 29 May 2014 20:52:01 +0000 Subject: [PATCH 29/34] HADOOP-10448. Support pluggable mechanism to specify proxy user settings (Contributed by Benoy Antony) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598396 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../fs/CommonConfigurationKeysPublic.java | 3 + .../DefaultImpersonationProvider.java | 210 +++++++++++++++++ .../authorize/ImpersonationProvider.java | 34 +++ .../hadoop/security/authorize/ProxyUsers.java | 213 +++--------------- .../src/main/resources/core-default.xml | 11 + .../apache/hadoop/ipc/MiniRPCBenchmark.java | 5 +- .../security/TestDoAsEffectiveUser.java | 15 +- .../security/authorize/TestProxyUsers.java | 117 +++++++--- .../apache/hadoop/hdfs/nfs/TestReaddir.java | 8 +- .../hadoop/hdfs/nfs/nfs3/TestWrites.java | 5 +- .../TestDelegationTokenForProxyUser.java | 5 +- .../hdfs/server/common/TestJspHelper.java | 5 +- .../hdfs/server/namenode/TestAuditLogger.java | 2 +- .../security/TestRefreshUserMappings.java | 5 +- .../resourcemanager/TestRMAdminService.java | 16 +- 16 files changed, 420 insertions(+), 237 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ImpersonationProvider.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 9a182057a89..e48d85e985a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -419,6 +419,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10618. Remove SingleNodeSetup.apt.vm. (Akira Ajisaka via Arpit Agarwal) + HADOOP-10448. Support pluggable mechanism to specify proxy user settings. + (Benoy Antony via Arpit Agarwal) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index b6be29447af..a2d7d96ceb7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -282,5 +282,8 @@ public class CommonConfigurationKeysPublic { /** Class to override Sasl Properties for a connection */ public static final String HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS = "hadoop.security.saslproperties.resolver.class"; + /** Class to override Impersonation provider */ + public static final String HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS = + "hadoop.security.impersonation.provider.class"; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java new file mode 100644 index 00000000000..46b4a928e85 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security.authorize; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.annotations.VisibleForTesting; + +public class DefaultImpersonationProvider implements ImpersonationProvider { + private static final String CONF_HOSTS = ".hosts"; + private static final String CONF_USERS = ".users"; + private static final String CONF_GROUPS = ".groups"; + private static final String CONF_HADOOP_PROXYUSER = "hadoop.proxyuser."; + private static final String CONF_HADOOP_PROXYUSER_RE = "hadoop\\.proxyuser\\."; + // list of users, groups and hosts per proxyuser + private Map> proxyUsers = + new HashMap>(); + private Map> proxyGroups = + new HashMap>(); + private Map> proxyHosts = + new HashMap>(); + private Configuration conf; + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + + // get all the new keys for users + String regex = CONF_HADOOP_PROXYUSER_RE+"[^.]*\\"+CONF_USERS; + Map allMatchKeys = conf.getValByRegex(regex); + for(Entry entry : allMatchKeys.entrySet()) { + Collection users = StringUtils.getTrimmedStringCollection(entry.getValue()); + proxyUsers.put(entry.getKey(), users); + } + + // get all the new keys for groups + regex = CONF_HADOOP_PROXYUSER_RE+"[^.]*\\"+CONF_GROUPS; + allMatchKeys = conf.getValByRegex(regex); + for(Entry entry : allMatchKeys.entrySet()) { + Collection groups = StringUtils.getTrimmedStringCollection(entry.getValue()); + proxyGroups.put(entry.getKey(), groups); + //cache the groups. This is needed for NetGroups + Groups.getUserToGroupsMappingService(conf).cacheGroupsAdd( + new ArrayList(groups)); + } + + // now hosts + regex = CONF_HADOOP_PROXYUSER_RE+"[^.]*\\"+CONF_HOSTS; + allMatchKeys = conf.getValByRegex(regex); + for(Entry entry : allMatchKeys.entrySet()) { + proxyHosts.put(entry.getKey(), + StringUtils.getTrimmedStringCollection(entry.getValue())); + } + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void authorize(UserGroupInformation user, + String remoteAddress) throws AuthorizationException { + + if (user.getRealUser() == null) { + return; + } + boolean userAuthorized = false; + boolean ipAuthorized = false; + UserGroupInformation superUser = user.getRealUser(); + + Collection allowedUsers = proxyUsers.get( + getProxySuperuserUserConfKey(superUser.getShortUserName())); + + if (isWildcardList(allowedUsers)) { + userAuthorized = true; + } else if (allowedUsers != null && !allowedUsers.isEmpty()) { + if (allowedUsers.contains(user.getShortUserName())) { + userAuthorized = true; + } + } + + if (!userAuthorized){ + Collection allowedUserGroups = proxyGroups.get( + getProxySuperuserGroupConfKey(superUser.getShortUserName())); + + if (isWildcardList(allowedUserGroups)) { + userAuthorized = true; + } else if (allowedUserGroups != null && !allowedUserGroups.isEmpty()) { + for (String group : user.getGroupNames()) { + if (allowedUserGroups.contains(group)) { + userAuthorized = true; + break; + } + } + } + + if (!userAuthorized) { + throw new AuthorizationException("User: " + superUser.getUserName() + + " is not allowed to impersonate " + user.getUserName()); + } + } + + Collection ipList = proxyHosts.get( + getProxySuperuserIpConfKey(superUser.getShortUserName())); + + if (isWildcardList(ipList)) { + ipAuthorized = true; + } else if (ipList != null && !ipList.isEmpty()) { + for (String allowedHost : ipList) { + InetAddress hostAddr; + try { + hostAddr = InetAddress.getByName(allowedHost); + } catch (UnknownHostException e) { + continue; + } + if (hostAddr.getHostAddress().equals(remoteAddress)) { + // Authorization is successful + ipAuthorized = true; + } + } + } + if(!ipAuthorized) { + throw new AuthorizationException("Unauthorized connection for super-user: " + + superUser.getUserName() + " from IP " + remoteAddress); + } + } + + /** + * Return true if the configuration specifies the special configuration value + * "*", indicating that any group or host list is allowed to use this configuration. + */ + private boolean isWildcardList(Collection list) { + return (list != null) && + (list.size() == 1) && + (list.contains("*")); + } + + /** + * Returns configuration key for effective usergroups allowed for a superuser + * + * @param userName name of the superuser + * @return configuration key for superuser usergroups + */ + public static String getProxySuperuserUserConfKey(String userName) { + return CONF_HADOOP_PROXYUSER+userName+CONF_USERS; + } + + /** + * Returns configuration key for effective groups allowed for a superuser + * + * @param userName name of the superuser + * @return configuration key for superuser groups + */ + public static String getProxySuperuserGroupConfKey(String userName) { + return CONF_HADOOP_PROXYUSER+userName+CONF_GROUPS; + } + + /** + * Return configuration key for superuser ip addresses + * + * @param userName name of the superuser + * @return configuration key for superuser ip-addresses + */ + public static String getProxySuperuserIpConfKey(String userName) { + return CONF_HADOOP_PROXYUSER+userName+CONF_HOSTS; + } + + @VisibleForTesting + public Map> getProxyUsers() { + return proxyUsers; + } + + @VisibleForTesting + public Map> getProxyGroups() { + return proxyGroups; + } + + @VisibleForTesting + public Map> getProxyHosts() { + return proxyHosts; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ImpersonationProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ImpersonationProvider.java new file mode 100644 index 00000000000..6e7a39565df --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ImpersonationProvider.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security.authorize; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.security.UserGroupInformation; + +public interface ImpersonationProvider extends Configurable { + /** + * Authorize the superuser which is doing doAs + * + * @param user ugi of the effective or proxy user which contains a real user + * @param remoteAddress the ip address of client + * @throws AuthorizationException + */ + public void authorize(UserGroupInformation user, String remoteAddress) + throws AuthorizationException; +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java index 128b572ac43..e221ae390f2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java @@ -18,42 +18,35 @@ package org.apache.hadoop.security.authorize; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.Groups; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.ReflectionUtils; import com.google.common.annotations.VisibleForTesting; @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce", "HBase", "Hive"}) public class ProxyUsers { - private static final String CONF_HOSTS = ".hosts"; - private static final String CONF_USERS = ".users"; - private static final String CONF_GROUPS = ".groups"; - private static final String CONF_HADOOP_PROXYUSER = "hadoop.proxyuser."; - private static final String CONF_HADOOP_PROXYUSER_RE = "hadoop\\.proxyuser\\."; - - private static boolean init = false; - //list of users, groups and hosts per proxyuser - private static Map> proxyUsers = - new HashMap>(); - private static Map> proxyGroups = - new HashMap>(); - private static Map> proxyHosts = - new HashMap>(); + private static volatile ImpersonationProvider sip ; /** - * reread the conf and get new values for "hadoop.proxyuser.*.groups/users/hosts" + * Returns an instance of ImpersonationProvider. + * Looks up the configuration to see if there is custom class specified. + * @param conf + * @return ImpersonationProvider + */ + private static ImpersonationProvider getInstance(Configuration conf) { + Class clazz = + conf.getClass( + CommonConfigurationKeysPublic.HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS, + DefaultImpersonationProvider.class, ImpersonationProvider.class); + return ReflectionUtils.newInstance(clazz, conf); + } + + /** + * refresh Impersonation rules */ public static void refreshSuperUserGroupsConfiguration() { //load server side configuration; @@ -64,73 +57,13 @@ public class ProxyUsers { * refresh configuration * @param conf */ - public static synchronized void refreshSuperUserGroupsConfiguration(Configuration conf) { - - // remove all existing stuff - proxyGroups.clear(); - proxyHosts.clear(); - proxyUsers.clear(); - - // get all the new keys for users - String regex = CONF_HADOOP_PROXYUSER_RE+"[^.]*\\"+CONF_USERS; - Map allMatchKeys = conf.getValByRegex(regex); - for(Entry entry : allMatchKeys.entrySet()) { - Collection users = StringUtils.getTrimmedStringCollection(entry.getValue()); - proxyUsers.put(entry.getKey(), users); - } - - // get all the new keys for groups - regex = CONF_HADOOP_PROXYUSER_RE+"[^.]*\\"+CONF_GROUPS; - allMatchKeys = conf.getValByRegex(regex); - for(Entry entry : allMatchKeys.entrySet()) { - Collection groups = StringUtils.getTrimmedStringCollection(entry.getValue()); - proxyGroups.put(entry.getKey(), groups ); - //cache the groups. This is needed for NetGroups - Groups.getUserToGroupsMappingService(conf).cacheGroupsAdd( - new ArrayList(groups)); - } - - // now hosts - regex = CONF_HADOOP_PROXYUSER_RE+"[^.]*\\"+CONF_HOSTS; - allMatchKeys = conf.getValByRegex(regex); - for(Entry entry : allMatchKeys.entrySet()) { - proxyHosts.put(entry.getKey(), - StringUtils.getTrimmedStringCollection(entry.getValue())); - } - init = true; + public static void refreshSuperUserGroupsConfiguration(Configuration conf) { + // sip is volatile. Any assignment to it as well as the object's state + // will be visible to all the other threads. + sip = getInstance(conf); ProxyServers.refresh(conf); } - /** - * Returns configuration key for effective users allowed for a superuser - * - * @param userName name of the superuser - * @return configuration key for superuser users - */ - public static String getProxySuperuserUserConfKey(String userName) { - return ProxyUsers.CONF_HADOOP_PROXYUSER+userName+ProxyUsers.CONF_USERS; - } - - /** - * Returns configuration key for effective user groups allowed for a superuser - * - * @param userName name of the superuser - * @return configuration key for superuser groups - */ - public static String getProxySuperuserGroupConfKey(String userName) { - return ProxyUsers.CONF_HADOOP_PROXYUSER+userName+ProxyUsers.CONF_GROUPS; - } - - /** - * Return configuration key for superuser ip addresses - * - * @param userName name of the superuser - * @return configuration key for superuser ip-addresses - */ - public static String getProxySuperuserIpConfKey(String userName) { - return ProxyUsers.CONF_HADOOP_PROXYUSER+userName+ProxyUsers.CONF_HOSTS; - } - /** * Authorize the superuser which is doing doAs * @@ -138,75 +71,14 @@ public class ProxyUsers { * @param remoteAddress the ip address of client * @throws AuthorizationException */ - public static synchronized void authorize(UserGroupInformation user, + public static void authorize(UserGroupInformation user, String remoteAddress) throws AuthorizationException { - - if(!init) { + if (sip==null) { + // In a race situation, It is possible for multiple threads to satisfy this condition. + // The last assignment will prevail. refreshSuperUserGroupsConfiguration(); } - - if (user.getRealUser() == null) { - return; - } - boolean userAuthorized = false; - boolean ipAuthorized = false; - UserGroupInformation superUser = user.getRealUser(); - - Collection allowedUsers = proxyUsers.get( - getProxySuperuserUserConfKey(superUser.getShortUserName())); - - if (isWildcardList(allowedUsers)) { - userAuthorized = true; - } else if (allowedUsers != null && !allowedUsers.isEmpty()) { - if (allowedUsers.contains(user.getShortUserName())) { - userAuthorized = true; - } - } - - if (!userAuthorized) { - Collection allowedUserGroups = proxyGroups.get( - getProxySuperuserGroupConfKey(superUser.getShortUserName())); - - if (isWildcardList(allowedUserGroups)) { - userAuthorized = true; - } else if (allowedUserGroups != null && !allowedUserGroups.isEmpty()) { - for (String group : user.getGroupNames()) { - if (allowedUserGroups.contains(group)) { - userAuthorized = true; - break; - } - } - } - - if (!userAuthorized) { - throw new AuthorizationException("User: " + superUser.getUserName() - + " is not allowed to impersonate " + user.getUserName()); - } - } - - Collection ipList = proxyHosts.get( - getProxySuperuserIpConfKey(superUser.getShortUserName())); - - if (isWildcardList(ipList)) { - ipAuthorized = true; - } else if (ipList != null && !ipList.isEmpty()) { - for (String allowedHost : ipList) { - InetAddress hostAddr; - try { - hostAddr = InetAddress.getByName(allowedHost); - } catch (UnknownHostException e) { - continue; - } - if (hostAddr.getHostAddress().equals(remoteAddress)) { - // Authorization is successful - ipAuthorized = true; - } - } - } - if (!ipAuthorized) { - throw new AuthorizationException("Unauthorized connection for super-user: " - + superUser.getUserName() + " from IP " + remoteAddress); - } + sip.authorize(user, remoteAddress); } /** @@ -218,33 +90,14 @@ public class ProxyUsers { * @deprecated use {@link #authorize(UserGroupInformation, String) instead. */ @Deprecated - public static synchronized void authorize(UserGroupInformation user, + public static void authorize(UserGroupInformation user, String remoteAddress, Configuration conf) throws AuthorizationException { authorize(user,remoteAddress); } - - /** - * Return true if the configuration specifies the special configuration value - * "*", indicating that any group or host list is allowed to use this configuration. - */ - private static boolean isWildcardList(Collection list) { - return (list != null) && - (list.size() == 1) && - (list.contains("*")); - } - - @VisibleForTesting - public static Map> getProxyUsers() { - return proxyUsers; - } - - @VisibleForTesting - public static Map> getProxyGroups() { - return proxyGroups; - } - - @VisibleForTesting - public static Map> getProxyHosts() { - return proxyHosts; + + @VisibleForTesting + public static DefaultImpersonationProvider getDefaultImpersonationProvider() { + return ((DefaultImpersonationProvider)sip); } + } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index ea0808eef75..f63b0af64e1 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -722,6 +722,17 @@ + + hadoop.security.impersonation.provider.class + + A class which implements ImpersonationProvider interface, used to + authorize whether one user can impersonate a specific user. + If not specified, the DefaultImpersonationProvider will be used. + If a class is specified, then that class will be used to determine + the impersonation capability. + + + hadoop.rpc.socket.factory.class.default org.apache.hadoop.net.StandardSocketFactory diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/MiniRPCBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/MiniRPCBenchmark.java index 576ce767487..0bc0b047dad 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/MiniRPCBenchmark.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/MiniRPCBenchmark.java @@ -35,6 +35,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenInfo; @@ -326,7 +327,7 @@ public class MiniRPCBenchmark { String shortUserName = UserGroupInformation.createRemoteUser(user).getShortUserName(); try { - conf.setStrings(ProxyUsers.getProxySuperuserGroupConfKey(shortUserName), + conf.setStrings(DefaultImpersonationProvider.getProxySuperuserGroupConfKey(shortUserName), GROUP_NAME_1); configureSuperUserIPAddresses(conf, shortUserName); // start the server @@ -410,7 +411,7 @@ public class MiniRPCBenchmark { } builder.append("127.0.1.1,"); builder.append(InetAddress.getLocalHost().getCanonicalHostName()); - conf.setStrings(ProxyUsers.getProxySuperuserIpConfKey(superUserShortName), + conf.setStrings(DefaultImpersonationProvider.getProxySuperuserIpConfKey(superUserShortName), builder.toString()); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java index 6626e8b6348..42e7881d3a9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java @@ -35,6 +35,7 @@ import org.apache.hadoop.ipc.Server; import org.apache.hadoop.ipc.VersionedProtocol; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenInfo; @@ -100,7 +101,7 @@ public class TestDoAsEffectiveUser { builder.append("127.0.1.1,"); builder.append(InetAddress.getLocalHost().getCanonicalHostName()); LOG.info("Local Ip addresses: "+builder.toString()); - conf.setStrings(ProxyUsers.getProxySuperuserIpConfKey(superUserShortName), + conf.setStrings(DefaultImpersonationProvider.getProxySuperuserIpConfKey(superUserShortName), builder.toString()); } @@ -180,7 +181,7 @@ public class TestDoAsEffectiveUser { @Test(timeout=4000) public void testRealUserSetup() throws IOException { final Configuration conf = new Configuration(); - conf.setStrings(ProxyUsers + conf.setStrings(DefaultImpersonationProvider .getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1"); configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME); Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class) @@ -213,7 +214,7 @@ public class TestDoAsEffectiveUser { public void testRealUserAuthorizationSuccess() throws IOException { final Configuration conf = new Configuration(); configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME); - conf.setStrings(ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), + conf.setStrings(DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1"); Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class) .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0) @@ -247,9 +248,9 @@ public class TestDoAsEffectiveUser { @Test public void testRealUserIPAuthorizationFailure() throws IOException { final Configuration conf = new Configuration(); - conf.setStrings(ProxyUsers.getProxySuperuserIpConfKey(REAL_USER_SHORT_NAME), + conf.setStrings(DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_SHORT_NAME), "20.20.20.20"); //Authorized IP address - conf.setStrings(ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), + conf.setStrings(DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1"); Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class) .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0) @@ -292,7 +293,7 @@ public class TestDoAsEffectiveUser { @Test public void testRealUserIPNotSpecified() throws IOException { final Configuration conf = new Configuration(); - conf.setStrings(ProxyUsers + conf.setStrings(DefaultImpersonationProvider .getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1"); Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class) .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0) @@ -376,7 +377,7 @@ public class TestDoAsEffectiveUser { public void testRealUserGroupAuthorizationFailure() throws IOException { final Configuration conf = new Configuration(); configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME); - conf.setStrings(ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), + conf.setStrings(DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group3"); Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class) .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java index f4775d6d826..dcb3e7c823c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.security.authorize; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + import java.io.IOException; import java.util.Arrays; import java.util.Collection; @@ -25,13 +28,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.util.NativeCodeLoader; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.UserGroupInformation; - +import org.apache.hadoop.util.NativeCodeLoader; +import org.apache.hadoop.util.StringUtils; import org.junit.Test; -import static org.junit.Assert.*; public class TestProxyUsers { @@ -46,6 +47,8 @@ public class TestProxyUsers { new String[] { "@foo_group" }; private static final String[] OTHER_GROUP_NAMES = new String[] { "bar_group" }; + private static final String[] SUDO_GROUP_NAMES = + new String[] { "sudo_proxied_user" }; private static final String PROXY_IP = "1.2.3.4"; /** @@ -106,10 +109,10 @@ public class TestProxyUsers { groupMappingClassName); conf.set( - ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_NAME), StringUtils.join(",", Arrays.asList(NETGROUP_NAMES))); conf.set( - ProxyUsers.getProxySuperuserIpConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_NAME), PROXY_IP); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); @@ -130,10 +133,10 @@ public class TestProxyUsers { public void testProxyUsers() throws Exception { Configuration conf = new Configuration(); conf.set( - ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_NAME), StringUtils.join(",", Arrays.asList(GROUP_NAMES))); conf.set( - ProxyUsers.getProxySuperuserIpConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_NAME), PROXY_IP); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); @@ -164,11 +167,11 @@ public class TestProxyUsers { public void testProxyUsersWithUserConf() throws Exception { Configuration conf = new Configuration(); conf.set( - ProxyUsers.getProxySuperuserUserConfKey(REAL_USER_NAME), - StringUtils.join(",", Arrays.asList(AUTHORIZED_PROXY_USER_NAME))); + DefaultImpersonationProvider.getProxySuperuserUserConfKey(REAL_USER_NAME), + StringUtils.join(",", Arrays.asList(AUTHORIZED_PROXY_USER_NAME))); conf.set( - ProxyUsers.getProxySuperuserIpConfKey(REAL_USER_NAME), - PROXY_IP); + DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_NAME), + PROXY_IP); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); @@ -198,10 +201,10 @@ public class TestProxyUsers { public void testWildcardGroup() { Configuration conf = new Configuration(); conf.set( - ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_NAME), "*"); conf.set( - ProxyUsers.getProxySuperuserIpConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_NAME), PROXY_IP); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); @@ -232,10 +235,10 @@ public class TestProxyUsers { public void testWildcardUser() { Configuration conf = new Configuration(); conf.set( - ProxyUsers.getProxySuperuserUserConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserUserConfKey(REAL_USER_NAME), "*"); conf.set( - ProxyUsers.getProxySuperuserIpConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_NAME), PROXY_IP); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); @@ -266,10 +269,10 @@ public class TestProxyUsers { public void testWildcardIP() { Configuration conf = new Configuration(); conf.set( - ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_NAME), StringUtils.join(",", Arrays.asList(GROUP_NAMES))); conf.set( - ProxyUsers.getProxySuperuserIpConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_NAME), "*"); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); @@ -297,15 +300,16 @@ public class TestProxyUsers { public void testWithDuplicateProxyGroups() throws Exception { Configuration conf = new Configuration(); conf.set( - ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_NAME), StringUtils.join(",", Arrays.asList(GROUP_NAMES,GROUP_NAMES))); conf.set( - ProxyUsers.getProxySuperuserIpConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_NAME), PROXY_IP); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); - Collection groupsToBeProxied = ProxyUsers.getProxyGroups().get( - ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER_NAME)); + Collection groupsToBeProxied = + ProxyUsers.getDefaultImpersonationProvider().getProxyGroups().get( + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_NAME)); assertEquals (1,groupsToBeProxied.size()); } @@ -314,18 +318,51 @@ public class TestProxyUsers { public void testWithDuplicateProxyHosts() throws Exception { Configuration conf = new Configuration(); conf.set( - ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_NAME), StringUtils.join(",", Arrays.asList(GROUP_NAMES))); conf.set( - ProxyUsers.getProxySuperuserIpConfKey(REAL_USER_NAME), + DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_NAME), StringUtils.join(",", Arrays.asList(PROXY_IP,PROXY_IP))); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); - Collection hosts = ProxyUsers.getProxyHosts().get( - ProxyUsers.getProxySuperuserIpConfKey(REAL_USER_NAME)); + Collection hosts = + ProxyUsers.getDefaultImpersonationProvider().getProxyHosts().get( + DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_NAME)); assertEquals (1,hosts.size()); } + + @Test + public void testProxyUsersWithProviderOverride() throws Exception { + Configuration conf = new Configuration(); + conf.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS, + "org.apache.hadoop.security.authorize.TestProxyUsers$TestDummyImpersonationProvider"); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); + + // First try proxying a group that's allowed + UserGroupInformation realUserUgi = UserGroupInformation + .createUserForTesting(REAL_USER_NAME, SUDO_GROUP_NAMES); + UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting( + PROXY_USER_NAME, realUserUgi, GROUP_NAMES); + + // From good IP + assertAuthorized(proxyUserUgi, "1.2.3.4"); + // From bad IP + assertAuthorized(proxyUserUgi, "1.2.3.5"); + + // Now try proxying a group that's not allowed + realUserUgi = UserGroupInformation + .createUserForTesting(REAL_USER_NAME, GROUP_NAMES); + proxyUserUgi = UserGroupInformation.createProxyUserForTesting( + PROXY_USER_NAME, realUserUgi, GROUP_NAMES); + + // From good IP + assertNotAuthorized(proxyUserUgi, "1.2.3.4"); + // From bad IP + assertNotAuthorized(proxyUserUgi, "1.2.3.5"); + } + private void assertNotAuthorized(UserGroupInformation proxyUgi, String host) { try { @@ -343,4 +380,32 @@ public class TestProxyUsers { fail("Did not allow authorization of " + proxyUgi + " from " + host); } } + + static class TestDummyImpersonationProvider implements ImpersonationProvider { + /** + * Authorize a user (superuser) to impersonate another user (user1) if the + * superuser belongs to the group "sudo_user1" . + */ + + public void authorize(UserGroupInformation user, + String remoteAddress) throws AuthorizationException{ + UserGroupInformation superUser = user.getRealUser(); + + String sudoGroupName = "sudo_" + user.getShortUserName(); + if (!Arrays.asList(superUser.getGroupNames()).contains(sudoGroupName)){ + throw new AuthorizationException("User: " + superUser.getUserName() + + " is not allowed to impersonate " + user.getUserName()); + } + } + + @Override + public void setConf(Configuration conf) { + + } + + @Override + public Configuration getConf() { + return null; + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java index 1ca43bc8abb..4a19c6e82ce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.net.InetAddress; -import java.util.Arrays; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -41,9 +40,8 @@ import org.apache.hadoop.nfs.nfs3.response.READDIRPLUS3Response; import org.apache.hadoop.nfs.nfs3.response.READDIRPLUS3Response.EntryPlus3; import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.SecurityHandler; -import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; -import org.apache.hadoop.util.StringUtils; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -67,10 +65,10 @@ public class TestReaddir { public static void setup() throws Exception { String currentUser = System.getProperty("user.name"); config.set( - ProxyUsers.getProxySuperuserGroupConfKey(currentUser), + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(currentUser), "*"); config.set( - ProxyUsers.getProxySuperuserIpConfKey(currentUser), + DefaultImpersonationProvider.getProxySuperuserIpConfKey(currentUser), "*"); ProxyUsers.refreshSuperUserGroupsConfiguration(config); cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java index 4771204a252..e1d7d4067ce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java @@ -50,6 +50,7 @@ import org.apache.hadoop.nfs.nfs3.response.CREATE3Response; import org.apache.hadoop.nfs.nfs3.response.READ3Response; import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.SecurityHandler; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.jboss.netty.channel.Channel; import org.junit.Assert; @@ -288,10 +289,10 @@ public class TestWrites { System.getProperty("user.name")); String currentUser = System.getProperty("user.name"); config.set( - ProxyUsers.getProxySuperuserGroupConfKey(currentUser), + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(currentUser), "*"); config.set( - ProxyUsers.getProxySuperuserIpConfKey(currentUser), + DefaultImpersonationProvider.getProxySuperuserIpConfKey(currentUser), "*"); ProxyUsers.refreshSuperUserGroupsConfiguration(config); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java index 05a89042577..fa29b4b03f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; import org.apache.hadoop.security.TestDoAsEffectiveUser; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.Token; import org.junit.AfterClass; @@ -88,7 +89,7 @@ public class TestDelegationTokenForProxyUser { builder.append("127.0.1.1,"); builder.append(InetAddress.getLocalHost().getCanonicalHostName()); LOG.info("Local Ip addresses: " + builder.toString()); - conf.setStrings(ProxyUsers.getProxySuperuserIpConfKey(superUserShortName), + conf.setStrings(DefaultImpersonationProvider.getProxySuperuserIpConfKey(superUserShortName), builder.toString()); } @@ -100,7 +101,7 @@ public class TestDelegationTokenForProxyUser { DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000); config.setLong( DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000); - config.setStrings(ProxyUsers.getProxySuperuserGroupConfKey(REAL_USER), + config.setStrings(DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER), "group1"); config.setBoolean( DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java index d8209e3f973..827a33aa5ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java @@ -30,6 +30,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyServers; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.token.Token; @@ -284,8 +285,8 @@ public class TestJspHelper { String user = "TheNurse"; conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - conf.set(ProxyUsers.getProxySuperuserGroupConfKey(realUser), "*"); - conf.set(ProxyUsers.getProxySuperuserIpConfKey(realUser), "*"); + conf.set(DefaultImpersonationProvider.getProxySuperuserGroupConfKey(realUser), "*"); + conf.set(DefaultImpersonationProvider.getProxySuperuserIpConfKey(realUser), "*"); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); UserGroupInformation.setConfiguration(conf); UserGroupInformation ugi; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java index 8d40cd01c5e..29fee68a06e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java @@ -39,8 +39,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.web.resources.GetOpParam; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.authorize.ProxyServers; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.junit.Before; import org.junit.Test; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java index df9462828be..aadc6b0a417 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java @@ -41,6 +41,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.junit.After; import org.junit.Before; @@ -150,8 +151,8 @@ public class TestRefreshUserMappings { final String [] GROUP_NAMES2 = new String [] {"gr3" , "gr4"}; //keys in conf - String userKeyGroups = ProxyUsers.getProxySuperuserGroupConfKey(SUPER_USER); - String userKeyHosts = ProxyUsers.getProxySuperuserIpConfKey (SUPER_USER); + String userKeyGroups = DefaultImpersonationProvider.getProxySuperuserGroupConfKey(SUPER_USER); + String userKeyHosts = DefaultImpersonationProvider.getProxySuperuserIpConfKey (SUPER_USER); config.set(userKeyGroups, "gr3,gr4,gr5"); // superuser can proxy for this group config.set(userKeyHosts,"127.0.0.1"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 32e78ebf182..554f57c8a0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -348,14 +348,14 @@ public class TestRMAdminService { rm.adminService.refreshSuperUserGroupsConfiguration( RefreshSuperUserGroupsConfigurationRequest.newInstance()); - Assert.assertTrue(ProxyUsers.getProxyGroups() + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups() .get("hadoop.proxyuser.test.groups").size() == 1); - Assert.assertTrue(ProxyUsers.getProxyGroups() + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups() .get("hadoop.proxyuser.test.groups").contains("test_groups")); - Assert.assertTrue(ProxyUsers.getProxyHosts() + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts() .get("hadoop.proxyuser.test.hosts").size() == 1); - Assert.assertTrue(ProxyUsers.getProxyHosts() + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts() .get("hadoop.proxyuser.test.hosts").contains("test_hosts")); } @@ -708,14 +708,14 @@ public class TestRMAdminService { aclsString); // verify ProxyUsers and ProxyHosts - Assert.assertTrue(ProxyUsers.getProxyGroups() + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups() .get("hadoop.proxyuser.test.groups").size() == 1); - Assert.assertTrue(ProxyUsers.getProxyGroups() + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups() .get("hadoop.proxyuser.test.groups").contains("test_groups")); - Assert.assertTrue(ProxyUsers.getProxyHosts() + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts() .get("hadoop.proxyuser.test.hosts").size() == 1); - Assert.assertTrue(ProxyUsers.getProxyHosts() + Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts() .get("hadoop.proxyuser.test.hosts").contains("test_hosts")); // verify UserToGroupsMappings From 1d5e564998356ba7a808abdb9872ed6370b4bc81 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 29 May 2014 21:01:55 +0000 Subject: [PATCH 30/34] HDFS-6461. Use Time#monotonicNow to compute duration in DataNode#shutDown. Contributed by James Thomas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598399 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/server/datanode/DataNode.java | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1fde8757897..a5980964a93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -607,6 +607,9 @@ Release 2.5.0 - UNRELEASED HDFS-6453. Use Time#monotonicNow to avoid system clock reset. (Liang Xie via wang) + HDFS-6461. Use Time#monotonicNow to compute duration in DataNode#shutDown. + (James Thomas via wang) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index bd0cb337d90..a2c3493c704 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -1221,7 +1221,7 @@ public class DataNode extends Configured } // Record the time of initial notification - long timeNotified = Time.now(); + long timeNotified = Time.monotonicNow(); if (localDataXceiverServer != null) { ((DataXceiverServer) this.localDataXceiverServer.getRunnable()).kill(); @@ -1253,8 +1253,9 @@ public class DataNode extends Configured while (true) { // When shutting down for restart, wait 2.5 seconds before forcing // termination of receiver threads. - if (!this.shutdownForUpgrade || - (this.shutdownForUpgrade && (Time.now() - timeNotified > 2500))) { + if (!this.shutdownForUpgrade || + (this.shutdownForUpgrade && (Time.monotonicNow() - timeNotified + > 2500))) { this.threadGroup.interrupt(); } LOG.info("Waiting for threadgroup to exit, active threads is " + From c55947343e0a629f71d69b49ba87997573fe48a2 Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Thu, 29 May 2014 21:16:04 +0000 Subject: [PATCH 31/34] HDFS-6462. NFS: fsstat request fails with the secure hdfs. Contributed by Brandon Li git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598405 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 12 +----------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java index 62aa8cbedc0..9c546c25068 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.nfs.AccessPrivilege; import org.apache.hadoop.nfs.NfsExports; @@ -152,13 +151,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface { private final NfsExports exports; - /** - * superUserClient should always impersonate HDFS file system owner to send - * requests which requires supergroup privilege. This requires the same user - * to start HDFS and NFS. - */ - private final DFSClient superUserClient; - private final short replication; private final long blockSize; private final int bufferSize; @@ -180,7 +172,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface { exports = NfsExports.getInstance(config); writeManager = new WriteManager(iug, config); clientCache = new DFSClientCache(config); - superUserClient = new DFSClient(NameNode.getAddress(config), config); replication = (short) config.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, DFSConfigKeys.DFS_REPLICATION_DEFAULT); blockSize = config.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, @@ -1681,8 +1672,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface { } try { - // Use superUserClient to get file system status - FsStatus fsStatus = superUserClient.getDiskStatus(); + FsStatus fsStatus = dfsClient.getDiskStatus(); long totalBytes = fsStatus.getCapacity(); long freeBytes = fsStatus.getRemaining(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a5980964a93..afb7ff4d8e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -610,6 +610,8 @@ Release 2.5.0 - UNRELEASED HDFS-6461. Use Time#monotonicNow to compute duration in DataNode#shutDown. (James Thomas via wang) + HDFS-6462. NFS: fsstat request fails with the secure hdfs (brandonli) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES From 5dff070e5194aae6fd70526cf7607b26d7ee5d89 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 29 May 2014 21:32:44 +0000 Subject: [PATCH 32/34] HDFS-6356. Fix typo in DatanodeLayoutVersion. Contributed by Tulasi G. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598408 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index afb7ff4d8e6..bb7a73bc50c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -449,6 +449,8 @@ Release 2.5.0 - UNRELEASED HDFS-6416. Use Time#monotonicNow in OpenFileCtx and OpenFileCtxCatch to avoid system clock bugs (Abhiraj Butala via brandonli) + HDFS-6356. Fix typo in DatanodeLayoutVersion. (Tulasi G via wang) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java index ee0999e0be8..26c7457645c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java @@ -62,7 +62,7 @@ public class DataNodeLayoutVersion { * */ public static enum Feature implements LayoutFeature { - FIRST_LAYOUT(-55, -53, "First datenode layout", false); + FIRST_LAYOUT(-55, -53, "First datanode layout", false); private final FeatureInfo info; From f40a36758bb183b23181fd59a99d9ab87a428166 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 29 May 2014 21:38:39 +0000 Subject: [PATCH 33/34] HDFS-6447. balancer should timestamp the completion message. Contributed by Juan Yu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598411 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/server/balancer/Balancer.java | 1 + 2 files changed, 4 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bb7a73bc50c..f6e8f269227 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -451,6 +451,9 @@ Release 2.5.0 - UNRELEASED HDFS-6356. Fix typo in DatanodeLayoutVersion. (Tulasi G via wang) + HDFS-6447. balancer should timestamp the completion message. + (Juan Yu via wang). + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index f6e05a0c6ed..8061b053568 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -1552,6 +1552,7 @@ public class Balancer { System.out.println(e + ". Exiting ..."); return ReturnStatus.INTERRUPTED.code; } finally { + System.out.format("%-24s ", DateFormat.getDateTimeInstance().format(new Date())); System.out.println("Balancing took " + time2Str(Time.now()-startTime)); } } From 04b0f6851bcfcd8986868f06d64e4339ef8609de Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 29 May 2014 21:39:16 +0000 Subject: [PATCH 34/34] HADOOP-10639. FileBasedKeyStoresFactory initialization is not using default for SSL_REQUIRE_CLIENT_CERT_KEY. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1598413 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../security/ssl/FileBasedKeyStoresFactory.java | 3 ++- .../apache/hadoop/security/ssl/TestSSLFactory.java | 13 +++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e48d85e985a..855dbd04679 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -516,6 +516,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10602. Documentation has broken "Go Back" hyperlinks. (Akira AJISAKA via cnauroth) + HADOOP-10639. FileBasedKeyStoresFactory initialization is not using default + for SSL_REQUIRE_CLIENT_CERT_KEY. (tucu) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java index ef4fad041ea..52e54addaa3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java @@ -131,7 +131,8 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory { throws IOException, GeneralSecurityException { boolean requireClientCert = - conf.getBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, true); + conf.getBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, + SSLFactory.DEFAULT_SSL_REQUIRE_CLIENT_CERT); // certificate store String keystoreType = diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java index 1711a742260..20585b15d05 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java @@ -272,4 +272,17 @@ public class TestSSLFactory { sslFactory.destroy(); } } + + @Test + public void testNoClientCertsInitialization() throws Exception { + Configuration conf = createConfiguration(false); + conf.unset(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY); + SSLFactory sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + try { + sslFactory.init(); + } finally { + sslFactory.destroy(); + } + } + }