Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-13052. WebHDFS: Add support for snasphot diff. Contributed by Lokesh Jain.

This commit is contained in:
Xiaoyu Yao 2018-02-23 16:56:29 -08:00
parent 329a4fdd07
commit 1e84e46f16
8 changed files with 201 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
View File

@ -87,6 +87,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
SET_STORAGE_POLICY("op_set_storagePolicy"),
SET_TIMES(CommonStatisticNames.OP_SET_TIMES),
SET_XATTR("op_set_xattr"),
GET_SNAPSHOT_DIFF("op_get_snapshot_diff"),
TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
UNSET_STORAGE_POLICY("op_unset_storage_policy");

4
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java
View File

@ -69,6 +69,10 @@ public class SnapshotDiffReport {
}
return null;
}
public static DiffType parseDiffType(String s){
return DiffType.valueOf(s.toUpperCase());
}
}
/**

49
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
View File

@ -37,6 +37,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@ -49,6 +50,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ChunkedArrayList;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.StringUtils;
@ -693,4 +695,51 @@ class JsonUtilClient {
encryptDataTransfer, trashInterval, type, keyProviderUri,
storagepolicyId);
}
public static SnapshotDiffReport toSnapshotDiffReport(final Map<?, ?> json) {
if (json == null) {
return null;
}
Map<?, ?> m =
(Map<?, ?>) json.get(SnapshotDiffReport.class.getSimpleName());
String snapshotRoot = (String) m.get("snapshotRoot");
String fromSnapshot = (String) m.get("fromSnapshot");
String toSnapshot = (String) m.get("toSnapshot");
List<SnapshotDiffReport.DiffReportEntry> diffList =
toDiffList(getList(m, "diffList"));
return new SnapshotDiffReport(snapshotRoot, fromSnapshot, toSnapshot,
diffList);
}
private static List<SnapshotDiffReport.DiffReportEntry> toDiffList(
List<?> objs) {
if (objs == null) {
return null;
}
List<SnapshotDiffReport.DiffReportEntry> diffList =
new ChunkedArrayList<>();
for (int i = 0; i < objs.size(); i++) {
diffList.add(toDiffReportEntry((Map<?, ?>) objs.get(i)));
}
return diffList;
}
private static SnapshotDiffReport.DiffReportEntry toDiffReportEntry(
Map<?, ?> json) {
if (json == null) {
return null;
}
SnapshotDiffReport.DiffType type =
SnapshotDiffReport.DiffType.parseDiffType((String) json.get("type"));
byte[] sourcePath = toByteArray((String) json.get("sourcePath"));
byte[] targetPath = toByteArray((String) json.get("targetPath"));
return new SnapshotDiffReport.DiffReportEntry(type, sourcePath, targetPath);
}
private static byte[] toByteArray(String str) {
if (str == null) {
return null;
}
return DFSUtilClient.string2Bytes(str);
}
}

15
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
View File

@ -96,6 +96,7 @@ import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FileEncryptionInfoProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
@ -1317,6 +1318,20 @@ public class WebHdfsFileSystem extends FileSystem
new SnapshotNameParam(snapshotNewName)).run();
}
public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir,
final String fromSnapshot, final String toSnapshot) throws IOException {
storageStatistics.incrementOpCounter(OpType.GET_SNAPSHOT_DIFF);
final HttpOpParam.Op op = GetOpParam.Op.GETSNAPSHOTDIFF;
return new FsPathResponseRunner<SnapshotDiffReport>(op, snapshotDir,
new OldSnapshotNameParam(fromSnapshot),
new SnapshotNameParam(toSnapshot)) {
@Override
SnapshotDiffReport decodeResponse(Map<?, ?> json) {
return JsonUtilClient.toSnapshotDiffReport(json);
}
}.run();
}
@Override
public boolean setReplication(final Path p, final short replication
) throws IOException {

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
View File

@ -47,7 +47,8 @@ public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
CHECKACCESS(false, HttpURLConnection.HTTP_OK),
LISTSTATUS_BATCH(false, HttpURLConnection.HTTP_OK),
GETSERVERDEFAULTS(false, HttpURLConnection.HTTP_OK);
GETSERVERDEFAULTS(false, HttpURLConnection.HTTP_OK),
GETSNAPSHOTDIFF(false, HttpURLConnection.HTTP_OK);
final boolean redirect;
final int expectedHttpResponseCode;

33
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
View File

@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
@ -922,6 +923,10 @@ public class NamenodeWebHdfsMethods {
final ExcludeDatanodesParam excludeDatanodes,
@QueryParam(FsActionParam.NAME) @DefaultValue(FsActionParam.DEFAULT)
final FsActionParam fsAction,
@QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT)
final SnapshotNameParam snapshotName,
@QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT)
final OldSnapshotNameParam oldSnapshotName,
@QueryParam(TokenKindParam.NAME) @DefaultValue(TokenKindParam.DEFAULT)
final TokenKindParam tokenKind,
@QueryParam(TokenServiceParam.NAME) @DefaultValue(TokenServiceParam.DEFAULT)
@ -932,8 +937,9 @@ public class NamenodeWebHdfsMethods {
final StartAfterParam startAfter
) throws IOException, InterruptedException {
return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length,
renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, fsAction,
tokenKind, tokenService, noredirect, startAfter);
renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes,
fsAction, snapshotName, oldSnapshotName, tokenKind, tokenService,
noredirect, startAfter);
}
/** Handle HTTP GET request. */
@ -968,6 +974,10 @@ public class NamenodeWebHdfsMethods {
final ExcludeDatanodesParam excludeDatanodes,
@QueryParam(FsActionParam.NAME) @DefaultValue(FsActionParam.DEFAULT)
final FsActionParam fsAction,
@QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT)
final SnapshotNameParam snapshotName,
@QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT)
final OldSnapshotNameParam oldSnapshotName,
@QueryParam(TokenKindParam.NAME) @DefaultValue(TokenKindParam.DEFAULT)
final TokenKindParam tokenKind,
@QueryParam(TokenServiceParam.NAME) @DefaultValue(TokenServiceParam.DEFAULT)
@ -980,15 +990,15 @@ public class NamenodeWebHdfsMethods {
init(ugi, delegation, username, doAsUser, path, op, offset, length,
renewer, bufferSize, xattrEncoding, excludeDatanodes, fsAction,
tokenKind, tokenService, startAfter);
snapshotName, oldSnapshotName, tokenKind, tokenService, startAfter);
return doAs(ugi, new PrivilegedExceptionAction<Response>() {
@Override
public Response run() throws IOException, URISyntaxException {
return get(ugi, delegation, username, doAsUser,
path.getAbsolutePath(), op, offset, length, renewer, bufferSize,
xattrNames, xattrEncoding, excludeDatanodes, fsAction, tokenKind,
tokenService, noredirect, startAfter);
return get(ugi, delegation, username, doAsUser, path.getAbsolutePath(),
op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding,
excludeDatanodes, fsAction, snapshotName, oldSnapshotName,
tokenKind, tokenService, noredirect, startAfter);
}
});
}
@ -1015,6 +1025,8 @@ public class NamenodeWebHdfsMethods {
final XAttrEncodingParam xattrEncoding,
final ExcludeDatanodesParam excludeDatanodes,
final FsActionParam fsAction,
final SnapshotNameParam snapshotName,
final OldSnapshotNameParam oldSnapshotName,
final TokenKindParam tokenKind,
final TokenServiceParam tokenService,
final NoRedirectParam noredirectParam,
@ -1185,6 +1197,13 @@ public class NamenodeWebHdfsMethods {
return Response.ok(serverDefaultsResponse)
.type(MediaType.APPLICATION_JSON).build();
}
case GETSNAPSHOTDIFF: {
SnapshotDiffReport diffReport =
cp.getSnapshotDiffReport(fullpath, oldSnapshotName.getValue(),
snapshotName.getValue());
final String js = JsonUtil.toJsonString(diffReport);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
default:
throw new UnsupportedOperationException(op + " is not supported");
}

33
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
View File

@ -494,4 +494,37 @@ public class JsonUtil {
m.put("defaultStoragePolicyId", serverDefaults.getDefaultStoragePolicyId());
return m;
}
public static String toJsonString(SnapshotDiffReport diffReport) {
return toJsonString(SnapshotDiffReport.class.getSimpleName(),
toJsonMap(diffReport));
}
private static Object toJsonMap(SnapshotDiffReport diffReport) {
final Map<String, Object> m = new TreeMap<String, Object>();
m.put("snapshotRoot", diffReport.getSnapshotRoot());
m.put("fromSnapshot", diffReport.getFromSnapshot());
m.put("toSnapshot", diffReport.getLaterSnapshotName());
Object[] diffList = new Object[diffReport.getDiffList().size()];
for (int i = 0; i < diffReport.getDiffList().size(); i++) {
diffList[i] = toJsonMap(diffReport.getDiffList().get(i));
}
m.put("diffList", diffList);
return m;
}
private static Object toJsonMap(
SnapshotDiffReport.DiffReportEntry diffReportEntry) {
final Map<String, Object> m = new TreeMap<String, Object>();
m.put("type", diffReportEntry.getType());
if (diffReportEntry.getSourcePath() != null) {
m.put("sourcePath",
DFSUtilClient.bytes2String(diffReportEntry.getSourcePath()));
}
if (diffReportEntry.getTargetPath() != null) {
m.put("targetPath",
DFSUtilClient.bytes2String(diffReportEntry.getTargetPath()));
}
return m;
}
}

72
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
View File

@ -73,6 +73,7 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@ -82,6 +83,9 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
@ -579,7 +583,7 @@ public class TestWebHDFS {
}
/**
* Test snapshot creation through WebHdfs
* Test snapshot creation through WebHdfs.
*/
@Test
public void testWebHdfsCreateSnapshot() throws Exception {
@ -665,6 +669,72 @@ public class TestWebHDFS {
}
}
/**
* Test snapshot diff through WebHdfs
*/
@Test
public void testWebHdfsSnapshotDiff() throws Exception {
MiniDFSCluster cluster = null;
final Configuration conf = WebHdfsTestUtil.createConf();
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
cluster.waitActive();
final DistributedFileSystem dfs = cluster.getFileSystem();
final WebHdfsFileSystem webHdfs = WebHdfsTestUtil
.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
final Path foo = new Path("/foo");
dfs.mkdirs(foo);
Path file0 = new Path(foo, "file0");
DFSTestUtil.createFile(dfs, file0, 100, (short) 1, 0);
Path file1 = new Path(foo, "file1");
DFSTestUtil.createFile(dfs, file1, 100, (short) 1, 0);
Path file2 = new Path(foo, "file2");
DFSTestUtil.createFile(dfs, file2, 100, (short) 1, 0);
dfs.allowSnapshot(foo);
webHdfs.createSnapshot(foo, "s1");
final Path s1path = SnapshotTestHelper.getSnapshotRoot(foo, "s1");
Assert.assertTrue(webHdfs.exists(s1path));
Path file3 = new Path(foo, "file3");
DFSTestUtil.createFile(dfs, file3, 100, (short) 1, 0);
DFSTestUtil.appendFile(dfs, file0, 100);
dfs.delete(file1, false);
Path file4 = new Path(foo, "file4");
dfs.rename(file2, file4);
webHdfs.createSnapshot(foo, "s2");
SnapshotDiffReport diffReport =
webHdfs.getSnapshotDiffReport(foo, "s1", "s2");
Assert.assertEquals("/foo", diffReport.getSnapshotRoot());
Assert.assertEquals("s1", diffReport.getFromSnapshot());
Assert.assertEquals("s2", diffReport.getLaterSnapshotName());
DiffReportEntry entry0 =
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes(""));
DiffReportEntry entry1 =
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file0"));
DiffReportEntry entry2 =
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file1"));
DiffReportEntry entry3 =
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("file2"),
DFSUtil.string2Bytes("file4"));
DiffReportEntry entry4 =
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file3"));
Assert.assertTrue(diffReport.getDiffList().contains(entry0));
Assert.assertTrue(diffReport.getDiffList().contains(entry1));
Assert.assertTrue(diffReport.getDiffList().contains(entry2));
Assert.assertTrue(diffReport.getDiffList().contains(entry3));
Assert.assertTrue(diffReport.getDiffList().contains(entry4));
Assert.assertEquals(diffReport.getDiffList().size(), 5);
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
@Test
public void testWebHdfsCreateNonRecursive() throws IOException, URISyntaxException {
MiniDFSCluster cluster = null;