HDFS-13315. Add a test for the issue reported in HDFS-11481 which is fixed by HDFS-10997. Contributed by Yongjun Zhang.
This commit is contained in:
parent
a3ed6f40e1
commit
6c63cc7d30
|
@ -106,6 +106,7 @@ public class SnapshotDiff extends Configured implements Tool {
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
String[] content = e.getLocalizedMessage().split("\n");
|
String[] content = e.getLocalizedMessage().split("\n");
|
||||||
System.err.println("snapshotDiff: " + content[0]);
|
System.err.println("snapshotDiff: " + content[0]);
|
||||||
|
e.printStackTrace(System.err);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
|
|
|
@ -121,6 +121,9 @@ import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
|
||||||
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
|
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
|
||||||
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
|
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||||
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||||
|
@ -2354,4 +2357,38 @@ public class DFSTestUtil {
|
||||||
}
|
}
|
||||||
return closedFiles;
|
return closedFiles;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check the correctness of the snapshotDiff report.
|
||||||
|
* Make sure all items in the passed entries are in the snapshotDiff
|
||||||
|
* report.
|
||||||
|
*/
|
||||||
|
public static void verifySnapshotDiffReport(DistributedFileSystem fs,
|
||||||
|
Path dir, String from, String to,
|
||||||
|
DiffReportEntry... entries) throws IOException {
|
||||||
|
SnapshotDiffReport report = fs.getSnapshotDiffReport(dir, from, to);
|
||||||
|
// reverse the order of from and to
|
||||||
|
SnapshotDiffReport inverseReport = fs
|
||||||
|
.getSnapshotDiffReport(dir, to, from);
|
||||||
|
LOG.info(report.toString());
|
||||||
|
LOG.info(inverseReport.toString() + "\n");
|
||||||
|
|
||||||
|
assertEquals(entries.length, report.getDiffList().size());
|
||||||
|
assertEquals(entries.length, inverseReport.getDiffList().size());
|
||||||
|
|
||||||
|
for (DiffReportEntry entry : entries) {
|
||||||
|
if (entry.getType() == DiffType.MODIFY) {
|
||||||
|
assertTrue(report.getDiffList().contains(entry));
|
||||||
|
assertTrue(inverseReport.getDiffList().contains(entry));
|
||||||
|
} else if (entry.getType() == DiffType.DELETE) {
|
||||||
|
assertTrue(report.getDiffList().contains(entry));
|
||||||
|
assertTrue(inverseReport.getDiffList().contains(
|
||||||
|
new DiffReportEntry(DiffType.CREATE, entry.getSourcePath())));
|
||||||
|
} else if (entry.getType() == DiffType.CREATE) {
|
||||||
|
assertTrue(report.getDiffList().contains(entry));
|
||||||
|
assertTrue(inverseReport.getDiffList().contains(
|
||||||
|
new DiffReportEntry(DiffType.DELETE, entry.getSourcePath())));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -75,6 +75,8 @@ import org.apache.hadoop.hdfs.protocol.EncryptionZone;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
|
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.EncryptionFaultInjector;
|
import org.apache.hadoop.hdfs.server.namenode.EncryptionFaultInjector;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager;
|
import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
|
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
|
||||||
|
@ -148,6 +150,7 @@ import javax.xml.parsers.SAXParser;
|
||||||
import javax.xml.parsers.SAXParserFactory;
|
import javax.xml.parsers.SAXParserFactory;
|
||||||
|
|
||||||
public class TestEncryptionZones {
|
public class TestEncryptionZones {
|
||||||
|
static final Logger LOG = Logger.getLogger(TestEncryptionZones.class);
|
||||||
|
|
||||||
protected Configuration conf;
|
protected Configuration conf;
|
||||||
private FileSystemTestHelper fsHelper;
|
private FileSystemTestHelper fsHelper;
|
||||||
|
@ -1495,6 +1498,44 @@ public class TestEncryptionZones {
|
||||||
contents, DFSTestUtil.readFile(fs, snapshottedZoneFile));
|
contents, DFSTestUtil.readFile(fs, snapshottedZoneFile));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check the correctness of the diff reports.
|
||||||
|
*/
|
||||||
|
private void verifyDiffReport(Path dir, String from, String to,
|
||||||
|
DiffReportEntry... entries) throws IOException {
|
||||||
|
DFSTestUtil.verifySnapshotDiffReport(fs, dir, from, to, entries);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test correctness of snapshotDiff for encryption zone.
|
||||||
|
* snapshtoDiff should work when the path parameter is prefixed with
|
||||||
|
* /.reserved/raw for path that's both snapshottable and encryption zone.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testSnapshotDiffOnEncryptionZones() throws Exception {
|
||||||
|
final String TEST_KEY2 = "testkey2";
|
||||||
|
DFSTestUtil.createKey(TEST_KEY2, cluster, conf);
|
||||||
|
|
||||||
|
final int len = 8196;
|
||||||
|
final Path zone = new Path("/zone");
|
||||||
|
final Path rawZone = new Path("/.reserved/raw/zone");
|
||||||
|
final Path zoneFile = new Path(zone, "zoneFile");
|
||||||
|
fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
|
||||||
|
dfsAdmin.allowSnapshot(zone);
|
||||||
|
dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
|
||||||
|
DFSTestUtil.createFile(fs, zoneFile, len, (short) 1, 0xFEED);
|
||||||
|
fs.createSnapshot(zone, "snap1");
|
||||||
|
fsWrapper.delete(zoneFile, true);
|
||||||
|
fs.createSnapshot(zone, "snap2");
|
||||||
|
verifyDiffReport(zone, "snap1", "snap2",
|
||||||
|
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
|
||||||
|
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("zoneFile")));
|
||||||
|
|
||||||
|
verifyDiffReport(rawZone, "snap1", "snap2",
|
||||||
|
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
|
||||||
|
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("zoneFile")));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verify symlinks can be created in encryption zones and that
|
* Verify symlinks can be created in encryption zones and that
|
||||||
* they function properly when the target is in the same
|
* they function properly when the target is in the same
|
||||||
|
|
|
@ -194,30 +194,7 @@ public class TestSnapshotDiffReport {
|
||||||
*/
|
*/
|
||||||
private void verifyDiffReport(Path dir, String from, String to,
|
private void verifyDiffReport(Path dir, String from, String to,
|
||||||
DiffReportEntry... entries) throws IOException {
|
DiffReportEntry... entries) throws IOException {
|
||||||
SnapshotDiffReport report = hdfs.getSnapshotDiffReport(dir, from, to);
|
DFSTestUtil.verifySnapshotDiffReport(hdfs, dir, from, to, entries);
|
||||||
// reverse the order of from and to
|
|
||||||
SnapshotDiffReport inverseReport = hdfs
|
|
||||||
.getSnapshotDiffReport(dir, to, from);
|
|
||||||
LOG.info(report.toString());
|
|
||||||
LOG.info(inverseReport.toString() + "\n");
|
|
||||||
|
|
||||||
assertEquals(entries.length, report.getDiffList().size());
|
|
||||||
assertEquals(entries.length, inverseReport.getDiffList().size());
|
|
||||||
|
|
||||||
for (DiffReportEntry entry : entries) {
|
|
||||||
if (entry.getType() == DiffType.MODIFY) {
|
|
||||||
assertTrue(report.getDiffList().contains(entry));
|
|
||||||
assertTrue(inverseReport.getDiffList().contains(entry));
|
|
||||||
} else if (entry.getType() == DiffType.DELETE) {
|
|
||||||
assertTrue(report.getDiffList().contains(entry));
|
|
||||||
assertTrue(inverseReport.getDiffList().contains(
|
|
||||||
new DiffReportEntry(DiffType.CREATE, entry.getSourcePath())));
|
|
||||||
} else if (entry.getType() == DiffType.CREATE) {
|
|
||||||
assertTrue(report.getDiffList().contains(entry));
|
|
||||||
assertTrue(inverseReport.getDiffList().contains(
|
|
||||||
new DiffReportEntry(DiffType.DELETE, entry.getSourcePath())));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
Loading…
Reference in New Issue