HDFS-6665. Add tests for XAttrs in combination with viewfs. Contributed by Stephen Chu.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1614545 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Wang
parent
7e54b1c6d9
commit
12241908ad
|
|
@ -326,6 +326,9 @@ Release 2.6.0 - UNRELEASED
|
|||
|
||||
HDFS-6739. Add getDatanodeStorageReport to ClientProtocol. (szetszwo)
|
||||
|
||||
HDFS-6665. Add tests for XAttrs in combination with viewfs.
|
||||
(Stephen Chu via wang)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
HDFS-6690. Deduplicate xattr names in memory. (wang)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,151 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.fs.viewfs;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.FileSystemTestHelper;
|
||||
import org.apache.hadoop.fs.FsConstants;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.assertArrayEquals;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* Verify XAttrs through ViewFileSystem functionality.
|
||||
*/
|
||||
public class TestViewFileSystemWithXAttrs {
|
||||
|
||||
private static MiniDFSCluster cluster;
|
||||
private static Configuration clusterConf = new Configuration();
|
||||
private static FileSystem fHdfs;
|
||||
private static FileSystem fHdfs2;
|
||||
private FileSystem fsView;
|
||||
private Configuration fsViewConf;
|
||||
private FileSystem fsTarget, fsTarget2;
|
||||
private Path targetTestRoot, targetTestRoot2, mountOnNn1, mountOnNn2;
|
||||
private FileSystemTestHelper fileSystemTestHelper =
|
||||
new FileSystemTestHelper("/tmp/TestViewFileSystemWithXAttrs");
|
||||
|
||||
// XAttrs
|
||||
protected static final String name1 = "user.a1";
|
||||
protected static final byte[] value1 = {0x31, 0x32, 0x33};
|
||||
protected static final String name2 = "user.a2";
|
||||
protected static final byte[] value2 = {0x37, 0x38, 0x39};
|
||||
|
||||
@BeforeClass
|
||||
public static void clusterSetupAtBeginning() throws IOException {
|
||||
cluster = new MiniDFSCluster.Builder(clusterConf)
|
||||
.nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
|
||||
.numDataNodes(2)
|
||||
.build();
|
||||
cluster.waitClusterUp();
|
||||
|
||||
fHdfs = cluster.getFileSystem(0);
|
||||
fHdfs2 = cluster.getFileSystem(1);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void ClusterShutdownAtEnd() throws Exception {
|
||||
cluster.shutdown();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
fsTarget = fHdfs;
|
||||
fsTarget2 = fHdfs2;
|
||||
targetTestRoot = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
|
||||
targetTestRoot2 = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget2);
|
||||
|
||||
fsTarget.delete(targetTestRoot, true);
|
||||
fsTarget2.delete(targetTestRoot2, true);
|
||||
fsTarget.mkdirs(targetTestRoot);
|
||||
fsTarget2.mkdirs(targetTestRoot2);
|
||||
|
||||
fsViewConf = ViewFileSystemTestSetup.createConfig();
|
||||
setupMountPoints();
|
||||
fsView = FileSystem.get(FsConstants.VIEWFS_URI, fsViewConf);
|
||||
}
|
||||
|
||||
private void setupMountPoints() {
|
||||
mountOnNn1 = new Path("/mountOnNn1");
|
||||
mountOnNn2 = new Path("/mountOnNn2");
|
||||
ConfigUtil.addLink(fsViewConf, mountOnNn1.toString(),
|
||||
targetTestRoot.toUri());
|
||||
ConfigUtil.addLink(fsViewConf, mountOnNn2.toString(),
|
||||
targetTestRoot2.toUri());
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
fsTarget.delete(fileSystemTestHelper.getTestRootPath(fsTarget), true);
|
||||
fsTarget2.delete(fileSystemTestHelper.getTestRootPath(fsTarget2), true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a ViewFileSystem wrapped over multiple federated NameNodes will
|
||||
* dispatch the XAttr operations to the correct NameNode.
|
||||
*/
|
||||
@Test
|
||||
public void testXAttrOnMountEntry() throws Exception {
|
||||
// Set XAttrs on the first namespace and verify they are correct
|
||||
fsView.setXAttr(mountOnNn1, name1, value1);
|
||||
fsView.setXAttr(mountOnNn1, name2, value2);
|
||||
assertEquals(2, fsView.getXAttrs(mountOnNn1).size());
|
||||
assertArrayEquals(value1, fsView.getXAttr(mountOnNn1, name1));
|
||||
assertArrayEquals(value2, fsView.getXAttr(mountOnNn1, name2));
|
||||
// Double-check by getting the XAttrs using FileSystem
|
||||
// instead of ViewFileSystem
|
||||
assertArrayEquals(value1, fHdfs.getXAttr(targetTestRoot, name1));
|
||||
assertArrayEquals(value2, fHdfs.getXAttr(targetTestRoot, name2));
|
||||
|
||||
// Paranoid check: verify the other namespace does not
|
||||
// have XAttrs set on the same path.
|
||||
assertEquals(0, fsView.getXAttrs(mountOnNn2).size());
|
||||
assertEquals(0, fHdfs2.getXAttrs(targetTestRoot2).size());
|
||||
|
||||
// Remove the XAttr entries on the first namespace
|
||||
fsView.removeXAttr(mountOnNn1, name1);
|
||||
fsView.removeXAttr(mountOnNn1, name2);
|
||||
assertEquals(0, fsView.getXAttrs(mountOnNn1).size());
|
||||
assertEquals(0, fHdfs.getXAttrs(targetTestRoot).size());
|
||||
|
||||
// Now set XAttrs on the second namespace
|
||||
fsView.setXAttr(mountOnNn2, name1, value1);
|
||||
fsView.setXAttr(mountOnNn2, name2, value2);
|
||||
assertEquals(2, fsView.getXAttrs(mountOnNn2).size());
|
||||
assertArrayEquals(value1, fsView.getXAttr(mountOnNn2, name1));
|
||||
assertArrayEquals(value2, fsView.getXAttr(mountOnNn2, name2));
|
||||
assertArrayEquals(value1, fHdfs2.getXAttr(targetTestRoot2, name1));
|
||||
assertArrayEquals(value2, fHdfs2.getXAttr(targetTestRoot2, name2));
|
||||
|
||||
fsView.removeXAttr(mountOnNn2, name1);
|
||||
fsView.removeXAttr(mountOnNn2, name2);
|
||||
assertEquals(0, fsView.getXAttrs(mountOnNn2).size());
|
||||
assertEquals(0, fHdfs2.getXAttrs(targetTestRoot2).size());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.fs.viewfs;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileContext;
|
||||
import org.apache.hadoop.fs.FileContextTestHelper;
|
||||
import org.apache.hadoop.fs.FsConstants;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.assertArrayEquals;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* Verify XAttrs through ViewFs functionality.
|
||||
*/
|
||||
public class TestViewFsWithXAttrs {
|
||||
|
||||
private static MiniDFSCluster cluster;
|
||||
private static Configuration clusterConf = new Configuration();
|
||||
private static FileContext fc, fc2;
|
||||
private FileContext fcView, fcTarget, fcTarget2;
|
||||
private Configuration fsViewConf;
|
||||
private Path targetTestRoot, targetTestRoot2, mountOnNn1, mountOnNn2;
|
||||
private FileContextTestHelper fileContextTestHelper =
|
||||
new FileContextTestHelper("/tmp/TestViewFsWithXAttrs");
|
||||
|
||||
// XAttrs
|
||||
protected static final String name1 = "user.a1";
|
||||
protected static final byte[] value1 = {0x31, 0x32, 0x33};
|
||||
protected static final String name2 = "user.a2";
|
||||
protected static final byte[] value2 = {0x37, 0x38, 0x39};
|
||||
|
||||
@BeforeClass
|
||||
public static void clusterSetupAtBeginning() throws IOException {
|
||||
cluster = new MiniDFSCluster.Builder(clusterConf)
|
||||
.nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
|
||||
.numDataNodes(2)
|
||||
.build();
|
||||
cluster.waitClusterUp();
|
||||
|
||||
fc = FileContext.getFileContext(cluster.getURI(0), clusterConf);
|
||||
fc2 = FileContext.getFileContext(cluster.getURI(1), clusterConf);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void ClusterShutdownAtEnd() throws Exception {
|
||||
cluster.shutdown();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
fcTarget = fc;
|
||||
fcTarget2 = fc2;
|
||||
targetTestRoot = fileContextTestHelper.getAbsoluteTestRootPath(fc);
|
||||
targetTestRoot2 = fileContextTestHelper.getAbsoluteTestRootPath(fc2);
|
||||
|
||||
fcTarget.delete(targetTestRoot, true);
|
||||
fcTarget2.delete(targetTestRoot2, true);
|
||||
fcTarget.mkdir(targetTestRoot, new FsPermission((short) 0750), true);
|
||||
fcTarget2.mkdir(targetTestRoot2, new FsPermission((short) 0750), true);
|
||||
|
||||
fsViewConf = ViewFileSystemTestSetup.createConfig();
|
||||
setupMountPoints();
|
||||
fcView = FileContext.getFileContext(FsConstants.VIEWFS_URI, fsViewConf);
|
||||
}
|
||||
|
||||
private void setupMountPoints() {
|
||||
mountOnNn1 = new Path("/mountOnNn1");
|
||||
mountOnNn2 = new Path("/mountOnNn2");
|
||||
ConfigUtil.addLink(fsViewConf, mountOnNn1.toString(), targetTestRoot.toUri());
|
||||
ConfigUtil.addLink(fsViewConf, mountOnNn2.toString(), targetTestRoot2.toUri());
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
fcTarget.delete(fileContextTestHelper.getTestRootPath(fcTarget), true);
|
||||
fcTarget2.delete(fileContextTestHelper.getTestRootPath(fcTarget2), true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a ViewFs wrapped over multiple federated NameNodes will
|
||||
* dispatch the XAttr operations to the correct NameNode.
|
||||
*/
|
||||
@Test
|
||||
public void testXAttrOnMountEntry() throws Exception {
|
||||
// Set XAttrs on the first namespace and verify they are correct
|
||||
fcView.setXAttr(mountOnNn1, name1, value1);
|
||||
fcView.setXAttr(mountOnNn1, name2, value2);
|
||||
assertEquals(2, fcView.getXAttrs(mountOnNn1).size());
|
||||
assertArrayEquals(value1, fcView.getXAttr(mountOnNn1, name1));
|
||||
assertArrayEquals(value2, fcView.getXAttr(mountOnNn1, name2));
|
||||
// Double-check by getting the XAttrs using FileSystem
|
||||
// instead of ViewFs
|
||||
assertArrayEquals(value1, fc.getXAttr(targetTestRoot, name1));
|
||||
assertArrayEquals(value2, fc.getXAttr(targetTestRoot, name2));
|
||||
|
||||
// Paranoid check: verify the other namespace does not
|
||||
// have XAttrs set on the same path.
|
||||
assertEquals(0, fcView.getXAttrs(mountOnNn2).size());
|
||||
assertEquals(0, fc2.getXAttrs(targetTestRoot2).size());
|
||||
|
||||
// Remove the XAttr entries on the first namespace
|
||||
fcView.removeXAttr(mountOnNn1, name1);
|
||||
fcView.removeXAttr(mountOnNn1, name2);
|
||||
assertEquals(0, fcView.getXAttrs(mountOnNn1).size());
|
||||
assertEquals(0, fc.getXAttrs(targetTestRoot).size());
|
||||
|
||||
// Now set XAttrs on the second namespace
|
||||
fcView.setXAttr(mountOnNn2, name1, value1);
|
||||
fcView.setXAttr(mountOnNn2, name2, value2);
|
||||
assertEquals(2, fcView.getXAttrs(mountOnNn2).size());
|
||||
assertArrayEquals(value1, fcView.getXAttr(mountOnNn2, name1));
|
||||
assertArrayEquals(value2, fcView.getXAttr(mountOnNn2, name2));
|
||||
assertArrayEquals(value1, fc2.getXAttr(targetTestRoot2, name1));
|
||||
assertArrayEquals(value2, fc2.getXAttr(targetTestRoot2, name2));
|
||||
|
||||
fcView.removeXAttr(mountOnNn2, name1);
|
||||
fcView.removeXAttr(mountOnNn2, name2);
|
||||
assertEquals(0, fcView.getXAttrs(mountOnNn2).size());
|
||||
assertEquals(0, fc2.getXAttrs(targetTestRoot2).size());
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue