HDFS-7795. Show warning if not all favored nodes were chosen by namenode. Contributed by Kihwal Lee.
This commit is contained in:
parent
78a7e8d3a6
commit
db6606223c
|
@ -644,6 +644,9 @@ Release 2.7.0 - UNRELEASED
|
||||||
|
|
||||||
HDFS-4266. BKJM: Separate write and ack quorum (Rakesh R via umamahesh)
|
HDFS-4266. BKJM: Separate write and ack quorum (Rakesh R via umamahesh)
|
||||||
|
|
||||||
|
HDFS-7795. Show warning if not all favored nodes were chosen by namenode
|
||||||
|
(kihwal)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
|
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
|
||||||
|
|
|
@ -35,6 +35,7 @@ import java.nio.channels.ClosedChannelException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
|
import java.util.HashSet;
|
||||||
import java.util.LinkedList;
|
import java.util.LinkedList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
@ -1443,7 +1444,7 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
ExtendedBlock blockCopy = new ExtendedBlock(block);
|
ExtendedBlock blockCopy = new ExtendedBlock(block);
|
||||||
blockCopy.setNumBytes(blockSize);
|
blockCopy.setNumBytes(blockSize);
|
||||||
|
|
||||||
boolean[] targetPinnings = getPinnings(nodes);
|
boolean[] targetPinnings = getPinnings(nodes, true);
|
||||||
// send the request
|
// send the request
|
||||||
new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken,
|
new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken,
|
||||||
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
|
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
|
||||||
|
@ -1537,19 +1538,28 @@ public class DFSOutputStream extends FSOutputSummer
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean[] getPinnings(DatanodeInfo[] nodes) {
|
private boolean[] getPinnings(DatanodeInfo[] nodes, boolean shouldLog) {
|
||||||
if (favoredNodes == null) {
|
if (favoredNodes == null) {
|
||||||
return null;
|
return null;
|
||||||
} else {
|
} else {
|
||||||
boolean[] pinnings = new boolean[nodes.length];
|
boolean[] pinnings = new boolean[nodes.length];
|
||||||
|
HashSet<String> favoredSet =
|
||||||
|
new HashSet<String>(Arrays.asList(favoredNodes));
|
||||||
for (int i = 0; i < nodes.length; i++) {
|
for (int i = 0; i < nodes.length; i++) {
|
||||||
pinnings[i] = false;
|
pinnings[i] = favoredSet.remove(nodes[i].getXferAddrWithHostname());
|
||||||
for (int j = 0; j < favoredNodes.length; j++) {
|
if (DFSClient.LOG.isDebugEnabled()) {
|
||||||
if (nodes[i].getXferAddrWithHostname().equals(favoredNodes[j])) {
|
DFSClient.LOG.debug(nodes[i].getXferAddrWithHostname() +
|
||||||
pinnings[i] = true;
|
" was chosen by name node (favored=" + pinnings[i] +
|
||||||
break;
|
").");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (shouldLog && !favoredSet.isEmpty()) {
|
||||||
|
// There is one or more favored nodes that were not allocated.
|
||||||
|
DFSClient.LOG.warn(
|
||||||
|
"These favored nodes were specified but not chosen: " +
|
||||||
|
favoredSet +
|
||||||
|
" Specified favored nodes: " + Arrays.toString(favoredNodes));
|
||||||
|
|
||||||
}
|
}
|
||||||
return pinnings;
|
return pinnings;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue