add missing file for HDFS-5950
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1573434 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Colin McCabe
parent
dd049a2f60
commit
6465b0b55f
|
|
@ -0,0 +1,332 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hdfs.server.datanode;
|
||||
|
||||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH;
|
||||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH_DEFAULT;
|
||||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS;
|
||||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.InvalidRequestException;
|
||||
import org.apache.hadoop.hdfs.ExtendedBlockId;
|
||||
import org.apache.hadoop.hdfs.ShortCircuitShm;
|
||||
import org.apache.hadoop.hdfs.ShortCircuitShm.ShmId;
|
||||
import org.apache.hadoop.hdfs.ShortCircuitShm.Slot;
|
||||
import org.apache.hadoop.hdfs.ShortCircuitShm.SlotId;
|
||||
import org.apache.hadoop.io.nativeio.NativeIO;
|
||||
import org.apache.hadoop.io.nativeio.SharedFileDescriptorFactory;
|
||||
import org.apache.hadoop.net.unix.DomainSocket;
|
||||
import org.apache.hadoop.net.unix.DomainSocketWatcher;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.collect.HashMultimap;
|
||||
|
||||
/*
|
||||
* Manages client short-circuit memory segments on the DataNode.
|
||||
*
|
||||
* DFSClients request shared memory segments from the DataNode. The
|
||||
* ShortCircuitRegistry generates and manages these segments. Each segment
|
||||
* has a randomly generated 128-bit ID which uniquely identifies it. The
|
||||
* segments each contain several "slots."
|
||||
*
|
||||
* Before performing a short-circuit read, DFSClients must request a pair of
|
||||
* file descriptors from the DataNode via the REQUEST_SHORT_CIRCUIT_FDS
|
||||
* operation. As part of this operation, DFSClients pass the ID of the shared
|
||||
* memory segment they would like to use to communicate information about this
|
||||
* replica, as well as the slot number within that segment they would like to
|
||||
* use. Slot allocation is always done by the client.
|
||||
*
|
||||
* Slots are used to track the state of the block on the both the client and
|
||||
* datanode. When this DataNode mlocks a block, the corresponding slots for the
|
||||
* replicas are marked as "anchorable". Anchorable blocks can be safely read
|
||||
* without verifying the checksum. This means that BlockReaderLocal objects
|
||||
* using these replicas can skip checksumming. It also means that we can do
|
||||
* zero-copy reads on these replicas (the ZCR interface has no way of
|
||||
* verifying checksums.)
|
||||
*
|
||||
* When a DN needs to munlock a block, it needs to first wait for the block to
|
||||
* be unanchored by clients doing a no-checksum read or a zero-copy read. The
|
||||
* DN also marks the block's slots as "unanchorable" to prevent additional
|
||||
* clients from initiating these operations in the future.
|
||||
*
|
||||
* The counterpart fo this class on the client is {@link DfsClientShmManager}.
|
||||
*/
|
||||
public class ShortCircuitRegistry {
|
||||
public static final Log LOG = LogFactory.getLog(ShortCircuitRegistry.class);
|
||||
|
||||
private static final int SHM_LENGTH = 8192;
|
||||
|
||||
private static class RegisteredShm extends ShortCircuitShm
|
||||
implements DomainSocketWatcher.Handler {
|
||||
private final ShortCircuitRegistry registry;
|
||||
|
||||
RegisteredShm(ShmId shmId, FileInputStream stream,
|
||||
ShortCircuitRegistry registry) throws IOException {
|
||||
super(shmId, stream);
|
||||
this.registry = registry;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean handle(DomainSocket sock) {
|
||||
synchronized (registry) {
|
||||
synchronized (this) {
|
||||
registry.removeShm(this);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void removeShm(ShortCircuitShm shm) {
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.debug("removing shm " + shm);
|
||||
}
|
||||
// Stop tracking the shmId.
|
||||
RegisteredShm removedShm = segments.remove(shm.getShmId());
|
||||
Preconditions.checkState(removedShm == shm,
|
||||
"failed to remove " + shm.getShmId());
|
||||
// Stop tracking the slots.
|
||||
for (Iterator<Slot> iter = shm.slotIterator(); iter.hasNext(); ) {
|
||||
Slot slot = iter.next();
|
||||
boolean removed = slots.remove(slot.getBlockId(), slot);
|
||||
Preconditions.checkState(removed);
|
||||
slot.makeInvalid();
|
||||
}
|
||||
// De-allocate the memory map and close the shared file.
|
||||
shm.free();
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether or not the registry is enabled.
|
||||
*/
|
||||
private boolean enabled;
|
||||
|
||||
/**
|
||||
* The factory which creates shared file descriptors.
|
||||
*/
|
||||
private final SharedFileDescriptorFactory shmFactory;
|
||||
|
||||
/**
|
||||
* A watcher which sends out callbacks when the UNIX domain socket
|
||||
* associated with a shared memory segment closes.
|
||||
*/
|
||||
private final DomainSocketWatcher watcher;
|
||||
|
||||
private final HashMap<ShmId, RegisteredShm> segments =
|
||||
new HashMap<ShmId, RegisteredShm>(0);
|
||||
|
||||
private final HashMultimap<ExtendedBlockId, Slot> slots =
|
||||
HashMultimap.create(0, 1);
|
||||
|
||||
public ShortCircuitRegistry(Configuration conf) throws IOException {
|
||||
boolean enabled = false;
|
||||
SharedFileDescriptorFactory shmFactory = null;
|
||||
DomainSocketWatcher watcher = null;
|
||||
try {
|
||||
if (!NativeIO.isAvailable()) {
|
||||
LOG.debug("Disabling ShortCircuitRegistry because NativeIO is " +
|
||||
"not available.");
|
||||
return;
|
||||
}
|
||||
String shmPath = conf.get(DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH,
|
||||
DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATH_DEFAULT);
|
||||
if (shmPath.isEmpty()) {
|
||||
LOG.info("Disabling ShortCircuitRegistry because shmPath was not set.");
|
||||
return;
|
||||
}
|
||||
int interruptCheck = conf.getInt(
|
||||
DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS,
|
||||
DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT);
|
||||
if (interruptCheck <= 0) {
|
||||
LOG.info("Disabling ShortCircuitRegistry because interruptCheckMs " +
|
||||
"was set to " + interruptCheck);
|
||||
return;
|
||||
}
|
||||
shmFactory =
|
||||
new SharedFileDescriptorFactory("HadoopShortCircuitShm_", shmPath);
|
||||
watcher = new DomainSocketWatcher(interruptCheck);
|
||||
enabled = true;
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("created new ShortCircuitRegistry with interruptCheck=" +
|
||||
interruptCheck + ", shmPath=" + shmPath);
|
||||
}
|
||||
} finally {
|
||||
this.enabled = enabled;
|
||||
this.shmFactory = shmFactory;
|
||||
this.watcher = watcher;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a block mlock event from the FsDatasetCache.
|
||||
*
|
||||
* @param blockId The block that was mlocked.
|
||||
*/
|
||||
public synchronized void processBlockMlockEvent(ExtendedBlockId blockId) {
|
||||
if (!enabled) return;
|
||||
Set<Slot> affectedSlots = slots.get(blockId);
|
||||
for (Slot slot : affectedSlots) {
|
||||
slot.makeAnchorable();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark any slots associated with this blockId as unanchorable.
|
||||
*
|
||||
* @param blockId The block ID.
|
||||
* @return True if we should allow the munlock request.
|
||||
*/
|
||||
public synchronized boolean processBlockMunlockRequest(
|
||||
ExtendedBlockId blockId) {
|
||||
if (!enabled) return true;
|
||||
boolean allowMunlock = true;
|
||||
Set<Slot> affectedSlots = slots.get(blockId);
|
||||
for (Slot slot : affectedSlots) {
|
||||
slot.makeUnanchorable();
|
||||
if (slot.isAnchored()) {
|
||||
allowMunlock = false;
|
||||
}
|
||||
}
|
||||
return allowMunlock;
|
||||
}
|
||||
|
||||
public static class NewShmInfo implements Closeable {
|
||||
public final ShmId shmId;
|
||||
public final FileInputStream stream;
|
||||
|
||||
NewShmInfo(ShmId shmId, FileInputStream stream) {
|
||||
this.shmId = shmId;
|
||||
this.stream = stream;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
stream.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a DFSClient request to create a new memory segment.
|
||||
*
|
||||
* @param clientName Client name as reported by the client.
|
||||
* @param sock The DomainSocket to associate with this memory
|
||||
* segment. When this socket is closed, or the
|
||||
* other side writes anything to the socket, the
|
||||
* segment will be closed. This can happen at any
|
||||
* time, including right after this function returns.
|
||||
* @return A NewShmInfo object. The caller must close the
|
||||
* NewShmInfo object once they are done with it.
|
||||
* @throws IOException If the new memory segment could not be created.
|
||||
*/
|
||||
public NewShmInfo createNewMemorySegment(String clientName,
|
||||
DomainSocket sock) throws IOException {
|
||||
NewShmInfo info = null;
|
||||
RegisteredShm shm = null;
|
||||
ShmId shmId = null;
|
||||
synchronized (this) {
|
||||
if (!enabled) {
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("createNewMemorySegment: ShortCircuitRegistry is " +
|
||||
"not enabled.");
|
||||
}
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
FileInputStream fis = null;
|
||||
try {
|
||||
do {
|
||||
shmId = ShmId.createRandom();
|
||||
} while (segments.containsKey(shmId));
|
||||
fis = shmFactory.createDescriptor(clientName, SHM_LENGTH);
|
||||
shm = new RegisteredShm(shmId, fis, this);
|
||||
} finally {
|
||||
if (shm == null) {
|
||||
IOUtils.closeQuietly(fis);
|
||||
}
|
||||
}
|
||||
info = new NewShmInfo(shmId, fis);
|
||||
segments.put(shmId, shm);
|
||||
}
|
||||
// Drop the registry lock to prevent deadlock.
|
||||
// After this point, RegisteredShm#handle may be called at any time.
|
||||
watcher.add(sock, shm);
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("createNewMemorySegment: created " + info.shmId);
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
public synchronized void registerSlot(ExtendedBlockId blockId, SlotId slotId)
|
||||
throws InvalidRequestException {
|
||||
if (!enabled) {
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("registerSlot: ShortCircuitRegistry is " +
|
||||
"not enabled.");
|
||||
}
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
ShmId shmId = slotId.getShmId();
|
||||
RegisteredShm shm = segments.get(shmId);
|
||||
if (shm == null) {
|
||||
throw new InvalidRequestException("there is no shared memory segment " +
|
||||
"registered with shmId " + shmId);
|
||||
}
|
||||
Slot slot = shm.registerSlot(slotId.getSlotIdx(), blockId);
|
||||
boolean added = slots.put(blockId, slot);
|
||||
Preconditions.checkState(added);
|
||||
}
|
||||
|
||||
public synchronized void unregisterSlot(SlotId slotId)
|
||||
throws InvalidRequestException {
|
||||
if (!enabled) {
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("unregisterSlot: ShortCircuitRegistry is " +
|
||||
"not enabled.");
|
||||
}
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
ShmId shmId = slotId.getShmId();
|
||||
RegisteredShm shm = segments.get(shmId);
|
||||
if (shm == null) {
|
||||
throw new InvalidRequestException("there is no shared memory segment " +
|
||||
"registered with shmId " + shmId);
|
||||
}
|
||||
Slot slot = shm.getSlot(slotId.getSlotIdx());
|
||||
slot.makeInvalid();
|
||||
shm.unregisterSlot(slotId.getSlotIdx());
|
||||
slots.remove(slot.getBlockId(), slot);
|
||||
}
|
||||
|
||||
public void shutdown() {
|
||||
synchronized (this) {
|
||||
if (!enabled) return;
|
||||
enabled = false;
|
||||
}
|
||||
IOUtils.closeQuietly(watcher);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue