HDFS-9721. Allow Delimited PB OIV tool to run upon fsimage that contains INodeReference. (Xiao Chen via lei)
(cherry picked from commit 9d494f0c0e
)
This commit is contained in:
parent
94950c15c2
commit
dfbd0d4df4
|
@ -894,6 +894,9 @@ Release 2.8.0 - UNRELEASED
|
|||
HDFS-9638. Improve DistCp Help and documentation.
|
||||
(Wei-Chiu Chuang via Yongjun Zhang)
|
||||
|
||||
HDFS-9721. Allow Delimited PB OIV tool to run upon fsimage that contains
|
||||
INodeReference. (Xiao Chen via lei)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
|
||||
|
|
|
@ -204,7 +204,7 @@ class FSImageLoader {
|
|||
return dirs;
|
||||
}
|
||||
|
||||
private static ImmutableList<Long> loadINodeReferenceSection(InputStream in)
|
||||
static ImmutableList<Long> loadINodeReferenceSection(InputStream in)
|
||||
throws IOException {
|
||||
LOG.info("Loading inode references");
|
||||
ImmutableList.Builder<Long> builder = ImmutableList.builder();
|
||||
|
|
|
@ -0,0 +1,28 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Signals that a snapshot is ignored.
|
||||
*/
|
||||
public class IgnoreSnapshotException extends IOException {
|
||||
public IgnoreSnapshotException() {
|
||||
}
|
||||
}
|
|
@ -84,6 +84,7 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter {
|
|||
inodeName.isEmpty() ? "/" : inodeName);
|
||||
buffer.append(path.toString());
|
||||
PermissionStatus p = null;
|
||||
boolean isDir = false;
|
||||
|
||||
switch (inode.getType()) {
|
||||
case FILE:
|
||||
|
@ -109,6 +110,7 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter {
|
|||
append(buffer, 0); // Num bytes.
|
||||
append(buffer, dir.getNsQuota());
|
||||
append(buffer, dir.getDsQuota());
|
||||
isDir = true;
|
||||
break;
|
||||
case SYMLINK:
|
||||
INodeSymlink s = inode.getSymlink();
|
||||
|
@ -126,9 +128,28 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter {
|
|||
break;
|
||||
}
|
||||
assert p != null;
|
||||
append(buffer, p.getPermission().toString());
|
||||
String dirString = isDir ? "d" : "-";
|
||||
append(buffer, dirString + p.getPermission().toString());
|
||||
append(buffer, p.getUserName());
|
||||
append(buffer, p.getGroupName());
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHeader() {
|
||||
StringBuffer buffer = new StringBuffer();
|
||||
buffer.append("Path");
|
||||
append(buffer, "Replication");
|
||||
append(buffer, "ModificationTime");
|
||||
append(buffer, "AccessTime");
|
||||
append(buffer, "PreferredBlockSize");
|
||||
append(buffer, "BlocksCount");
|
||||
append(buffer, "FileSize");
|
||||
append(buffer, "NSQUOTA");
|
||||
append(buffer, "DSQUOTA");
|
||||
append(buffer, "Permission");
|
||||
append(buffer, "UserName");
|
||||
append(buffer, "GroupName");
|
||||
return buffer.toString();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Lists;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
|
@ -195,13 +196,17 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
dirMap.put(p.getId(), dir);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getParentPath(long inode) throws IOException {
|
||||
if (inode == INodeId.ROOT_INODE_ID) {
|
||||
return "";
|
||||
}
|
||||
Dir parent = dirChildMap.get(inode);
|
||||
Preconditions.checkState(parent != null,
|
||||
"Can not find parent directory for INode: %s", inode);
|
||||
if (parent == null) {
|
||||
// The inode is an INodeReference, which is generated from snapshot.
|
||||
// For delimited oiv tool, no need to print out metadata in snapshots.
|
||||
PBImageTextWriter.ignoreSnapshotName(inode);
|
||||
}
|
||||
return parent.getPath();
|
||||
}
|
||||
|
||||
|
@ -353,16 +358,22 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
return "/";
|
||||
}
|
||||
byte[] bytes = dirChildMap.get(toBytes(inode));
|
||||
Preconditions.checkState(bytes != null && bytes.length == 8,
|
||||
"Can not find parent directory for inode %s, "
|
||||
+ "fsimage might be corrupted", inode);
|
||||
if (bytes == null) {
|
||||
// The inode is an INodeReference, which is generated from snapshot.
|
||||
// For delimited oiv tool, no need to print out metadata in snapshots.
|
||||
PBImageTextWriter.ignoreSnapshotName(inode);
|
||||
}
|
||||
if (bytes.length != 8) {
|
||||
throw new IOException(
|
||||
"bytes array length error. Actual length is " + bytes.length);
|
||||
}
|
||||
long parent = toLong(bytes);
|
||||
if (!dirPathCache.containsKey(parent)) {
|
||||
bytes = dirMap.get(toBytes(parent));
|
||||
if (parent != INodeId.ROOT_INODE_ID) {
|
||||
Preconditions.checkState(bytes != null,
|
||||
"Can not find parent directory for inode %s, "
|
||||
+ ", the fsimage might be corrupted.", parent);
|
||||
if (parent != INodeId.ROOT_INODE_ID && bytes == null) {
|
||||
// The parent is an INodeReference, which is generated from snapshot.
|
||||
// For delimited oiv tool, no need to print out metadata in snapshots.
|
||||
PBImageTextWriter.ignoreSnapshotName(parent);
|
||||
}
|
||||
String parentName = toString(bytes);
|
||||
String parentPath =
|
||||
|
@ -401,6 +412,7 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
out.flush();
|
||||
IOUtils.cleanup(null, metadataMap);
|
||||
}
|
||||
|
||||
|
@ -411,6 +423,11 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
*/
|
||||
abstract protected String getEntry(String parent, INode inode);
|
||||
|
||||
/**
|
||||
* Get text output for the header line.
|
||||
*/
|
||||
abstract protected String getHeader();
|
||||
|
||||
public void visit(RandomAccessFile file) throws IOException {
|
||||
Configuration conf = new Configuration();
|
||||
if (!FSImageUtil.checkFileFormat(file)) {
|
||||
|
@ -442,6 +459,7 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
}
|
||||
});
|
||||
|
||||
ImmutableList<Long> refIdList = null;
|
||||
for (FileSummary.Section section : sections) {
|
||||
fin.getChannel().position(section.getOffset());
|
||||
is = FSImageUtil.wrapInputStreamForCompression(conf,
|
||||
|
@ -449,15 +467,22 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
fin, section.getLength())));
|
||||
switch (SectionName.fromString(section.getName())) {
|
||||
case STRING_TABLE:
|
||||
LOG.info("Loading string table");
|
||||
stringTable = FSImageLoader.loadStringTable(is);
|
||||
break;
|
||||
case INODE_REFERENCE:
|
||||
// Load INodeReference so that all INodes can be processed.
|
||||
// Snapshots are not handled and will just be ignored for now.
|
||||
LOG.info("Loading inode references");
|
||||
refIdList = FSImageLoader.loadINodeReferenceSection(is);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
loadDirectories(fin, sections, summary, conf);
|
||||
loadINodeDirSection(fin, sections, summary, conf);
|
||||
loadINodeDirSection(fin, sections, summary, conf, refIdList);
|
||||
metadataMap.sync();
|
||||
output(conf, summary, fin, sections);
|
||||
}
|
||||
|
@ -468,6 +493,7 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
throws IOException {
|
||||
InputStream is;
|
||||
long startTime = Time.monotonicNow();
|
||||
out.println(getHeader());
|
||||
for (FileSummary.Section section : sections) {
|
||||
if (SectionName.fromString(section.getName()) == SectionName.INODE) {
|
||||
fin.getChannel().position(section.getOffset());
|
||||
|
@ -508,7 +534,7 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
|
||||
private void loadINodeDirSection(
|
||||
FileInputStream fin, List<FileSummary.Section> sections,
|
||||
FileSummary summary, Configuration conf)
|
||||
FileSummary summary, Configuration conf, List<Long> refIdList)
|
||||
throws IOException {
|
||||
LOG.info("Loading INode directory section.");
|
||||
long startTime = Time.monotonicNow();
|
||||
|
@ -519,7 +545,7 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
InputStream is = FSImageUtil.wrapInputStreamForCompression(conf,
|
||||
summary.getCodec(), new BufferedInputStream(
|
||||
new LimitInputStream(fin, section.getLength())));
|
||||
buildNamespace(is);
|
||||
buildNamespace(is, refIdList);
|
||||
}
|
||||
}
|
||||
long timeTaken = Time.monotonicNow() - startTime;
|
||||
|
@ -549,7 +575,8 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
/**
|
||||
* Scan the INodeDirectory section to construct the namespace.
|
||||
*/
|
||||
private void buildNamespace(InputStream in) throws IOException {
|
||||
private void buildNamespace(InputStream in, List<Long> refIdList)
|
||||
throws IOException {
|
||||
int count = 0;
|
||||
while (true) {
|
||||
FsImageProto.INodeDirectorySection.DirEntry e =
|
||||
|
@ -562,12 +589,15 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
LOG.debug("Scanned {} directories.", count);
|
||||
}
|
||||
long parentId = e.getParent();
|
||||
// Referred INode is not support for now.
|
||||
for (int i = 0; i < e.getChildrenCount(); i++) {
|
||||
long childId = e.getChildren(i);
|
||||
metadataMap.putDirChild(parentId, childId);
|
||||
}
|
||||
Preconditions.checkState(e.getRefChildrenCount() == 0);
|
||||
for (int i = e.getChildrenCount();
|
||||
i < e.getChildrenCount() + e.getRefChildrenCount(); i++) {
|
||||
int refId = e.getRefChildren(i - e.getChildrenCount());
|
||||
metadataMap.putDirChild(parentId, refIdList.get(refId));
|
||||
}
|
||||
}
|
||||
LOG.info("Scanned {} INode directories to build namespace.", count);
|
||||
}
|
||||
|
@ -575,15 +605,41 @@ abstract class PBImageTextWriter implements Closeable {
|
|||
private void outputINodes(InputStream in) throws IOException {
|
||||
INodeSection s = INodeSection.parseDelimitedFrom(in);
|
||||
LOG.info("Found {} INodes in the INode section", s.getNumInodes());
|
||||
long ignored = 0;
|
||||
long ignoredSnapshots = 0;
|
||||
for (int i = 0; i < s.getNumInodes(); ++i) {
|
||||
INode p = INode.parseDelimitedFrom(in);
|
||||
String parentPath = metadataMap.getParentPath(p.getId());
|
||||
out.println(getEntry(parentPath, p));
|
||||
try {
|
||||
String parentPath = metadataMap.getParentPath(p.getId());
|
||||
out.println(getEntry(parentPath, p));
|
||||
} catch (IOException ioe) {
|
||||
ignored++;
|
||||
if (!(ioe instanceof IgnoreSnapshotException)) {
|
||||
LOG.warn("Exception caught, ignoring node:{}", p.getId(), ioe);
|
||||
} else {
|
||||
ignoredSnapshots++;
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Exception caught, ignoring node:{}.", p.getId(), ioe);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (LOG.isDebugEnabled() && i % 100000 == 0) {
|
||||
LOG.debug("Outputted {} INodes.", i);
|
||||
}
|
||||
}
|
||||
if (ignored > 0) {
|
||||
LOG.warn("Ignored {} nodes, including {} in snapshots. Please turn on"
|
||||
+ " debug log for details", ignored, ignoredSnapshots);
|
||||
}
|
||||
LOG.info("Outputted {} INodes.", s.getNumInodes());
|
||||
}
|
||||
|
||||
static void ignoreSnapshotName(long inode) throws IOException {
|
||||
// Ignore snapshots - we want the output similar to -ls -R.
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("No snapshot name found for inode {}", inode);
|
||||
}
|
||||
throw new IgnoreSnapshotException();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -87,6 +87,7 @@ public class TestOfflineImageViewer {
|
|||
|
||||
// namespace as written to dfs, to be compared with viewer's output
|
||||
final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
|
||||
static int dirCount = 0;
|
||||
|
||||
@Rule
|
||||
public TemporaryFolder folder = new TemporaryFolder();
|
||||
|
@ -113,7 +114,7 @@ public class TestOfflineImageViewer {
|
|||
DistributedFileSystem hdfs = cluster.getFileSystem();
|
||||
|
||||
// Create a reasonable namespace
|
||||
for (int i = 0; i < NUM_DIRS; i++) {
|
||||
for (int i = 0; i < NUM_DIRS; i++, dirCount++) {
|
||||
Path dir = new Path("/dir" + i);
|
||||
hdfs.mkdirs(dir);
|
||||
writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString()));
|
||||
|
@ -131,11 +132,13 @@ public class TestOfflineImageViewer {
|
|||
// Create an empty directory
|
||||
Path emptydir = new Path("/emptydir");
|
||||
hdfs.mkdirs(emptydir);
|
||||
dirCount++;
|
||||
writtenFiles.put(emptydir.toString(), hdfs.getFileStatus(emptydir));
|
||||
|
||||
//Create a directory whose name should be escaped in XML
|
||||
Path invalidXMLDir = new Path("/dirContainingInvalidXMLChar\u0000here");
|
||||
hdfs.mkdirs(invalidXMLDir);
|
||||
dirCount++;
|
||||
|
||||
// Get delegation tokens so we log the delegation token op
|
||||
Token<?>[] delegationTokens = hdfs
|
||||
|
@ -144,15 +147,24 @@ public class TestOfflineImageViewer {
|
|||
LOG.debug("got token " + t);
|
||||
}
|
||||
|
||||
final Path snapshot = new Path("/snapshot");
|
||||
hdfs.mkdirs(snapshot);
|
||||
hdfs.allowSnapshot(snapshot);
|
||||
hdfs.mkdirs(new Path("/snapshot/1"));
|
||||
hdfs.delete(snapshot, true);
|
||||
// Create INodeReference
|
||||
final Path src = new Path("/src");
|
||||
hdfs.mkdirs(src);
|
||||
dirCount++;
|
||||
writtenFiles.put(src.toString(), hdfs.getFileStatus(src));
|
||||
final Path orig = new Path("/src/orig");
|
||||
hdfs.mkdirs(orig);
|
||||
hdfs.allowSnapshot(src);
|
||||
hdfs.createSnapshot(src, "snapshot");
|
||||
final Path dst = new Path("/dst");
|
||||
hdfs.rename(orig, dst);
|
||||
dirCount++;
|
||||
writtenFiles.put(dst.toString(), hdfs.getFileStatus(dst));
|
||||
|
||||
// Set XAttrs so the fsimage contains XAttr ops
|
||||
final Path xattr = new Path("/xattr");
|
||||
hdfs.mkdirs(xattr);
|
||||
dirCount++;
|
||||
hdfs.setXAttr(xattr, "user.a1", new byte[]{ 0x31, 0x32, 0x33 });
|
||||
hdfs.setXAttr(xattr, "user.a2", new byte[]{ 0x37, 0x38, 0x39 });
|
||||
// OIV should be able to handle empty value XAttrs
|
||||
|
@ -232,8 +244,8 @@ public class TestOfflineImageViewer {
|
|||
matcher = p.matcher(outputString);
|
||||
assertTrue(matcher.find() && matcher.groupCount() == 1);
|
||||
int totalDirs = Integer.parseInt(matcher.group(1));
|
||||
// totalDirs includes root directory, empty directory, and xattr directory
|
||||
assertEquals(NUM_DIRS + 4, totalDirs);
|
||||
// totalDirs includes root directory
|
||||
assertEquals(dirCount + 1, totalDirs);
|
||||
|
||||
FileStatus maxFile = Collections.max(writtenFiles.values(),
|
||||
new Comparator<FileStatus>() {
|
||||
|
@ -285,7 +297,7 @@ public class TestOfflineImageViewer {
|
|||
|
||||
// verify the number of directories
|
||||
FileStatus[] statuses = webhdfs.listStatus(new Path("/"));
|
||||
assertEquals(NUM_DIRS + 3, statuses.length); // contains empty and xattr directory
|
||||
assertEquals(dirCount, statuses.length);
|
||||
|
||||
// verify the number of files in the directory
|
||||
statuses = webhdfs.listStatus(new Path("/dir0"));
|
||||
|
@ -393,11 +405,15 @@ public class TestOfflineImageViewer {
|
|||
BufferedReader reader =
|
||||
new BufferedReader(new InputStreamReader(input))) {
|
||||
String line;
|
||||
boolean header = true;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
System.out.println(line);
|
||||
String[] fields = line.split(DELIMITER);
|
||||
assertEquals(12, fields.length);
|
||||
fileNames.add(fields[0]);
|
||||
if (!header) {
|
||||
fileNames.add(fields[0]);
|
||||
}
|
||||
header = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue