HDFS-12792. RBF: Test Router-based federation using HDFSContract. Contributed by Inigo Goiri.
This commit is contained in:
parent
f738d75a86
commit
e196d158a2
|
@ -0,0 +1,98 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
|
||||||
|
import org.apache.hadoop.fs.contract.hdfs.HDFSContract;
|
||||||
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||||
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
|
||||||
|
import org.junit.Assert;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The contract of Router-based Federated HDFS.
|
||||||
|
*/
|
||||||
|
public class RouterHDFSContract extends HDFSContract {
|
||||||
|
|
||||||
|
public static final int BLOCK_SIZE =
|
||||||
|
AbstractFSContractTestBase.TEST_FILE_LEN;
|
||||||
|
private static MiniRouterDFSCluster cluster;
|
||||||
|
|
||||||
|
public RouterHDFSContract(Configuration conf) {
|
||||||
|
super(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
try {
|
||||||
|
cluster = new MiniRouterDFSCluster(true, 2);
|
||||||
|
|
||||||
|
// Start NNs and DNs and wait until ready
|
||||||
|
cluster.startCluster();
|
||||||
|
|
||||||
|
// Start routers with only an RPC service
|
||||||
|
cluster.startRouters();
|
||||||
|
|
||||||
|
// Register and verify all NNs with all routers
|
||||||
|
cluster.registerNamenodes();
|
||||||
|
cluster.waitNamenodeRegistration();
|
||||||
|
|
||||||
|
// Setup the mount table
|
||||||
|
cluster.installMockLocations();
|
||||||
|
|
||||||
|
// Making one Namenodes active per nameservice
|
||||||
|
if (cluster.isHighAvailability()) {
|
||||||
|
for (String ns : cluster.getNameservices()) {
|
||||||
|
cluster.switchToActive(ns, NAMENODES[0]);
|
||||||
|
cluster.switchToStandby(ns, NAMENODES[1]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cluster.waitActiveNamespaces();
|
||||||
|
} catch (Exception e) {
|
||||||
|
destroyCluster();
|
||||||
|
throw new IOException("Cannot start federated cluster", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void destroyCluster() throws IOException {
|
||||||
|
if (cluster != null) {
|
||||||
|
cluster.shutdown();
|
||||||
|
cluster = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static MiniDFSCluster getCluster() {
|
||||||
|
return cluster.getCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static FileSystem getFileSystem() throws IOException {
|
||||||
|
//assumes cluster is not null
|
||||||
|
Assert.assertNotNull("cluster not created", cluster);
|
||||||
|
return cluster.getRandomRouter().getFileSystem();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public FileSystem getTestFileSystem() throws IOException {
|
||||||
|
return getFileSystem();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,44 @@
|
||||||
|
/*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License. See accompanying LICENSE file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractContractAppendTest;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test append operations on the Router-based FS.
|
||||||
|
*/
|
||||||
|
public class TestRouterHDFSContractAppend extends AbstractContractAppendTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
RouterHDFSContract.createCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void teardownCluster() throws IOException {
|
||||||
|
RouterHDFSContract.destroyCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AbstractFSContract createContract(Configuration conf) {
|
||||||
|
return new RouterHDFSContract(conf);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,51 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractContractConcatTest;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test concat operations on the Router-based FS.
|
||||||
|
*/
|
||||||
|
public class TestRouterHDFSContractConcat extends AbstractContractConcatTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
RouterHDFSContract.createCluster();
|
||||||
|
// perform a simple operation on the cluster to verify it is up
|
||||||
|
RouterHDFSContract.getFileSystem().getDefaultBlockSize(new Path("/"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void teardownCluster() throws IOException {
|
||||||
|
RouterHDFSContract.destroyCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AbstractFSContract createContract(Configuration conf) {
|
||||||
|
return new RouterHDFSContract(conf);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,48 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test create operations on the Router-based FS.
|
||||||
|
*/
|
||||||
|
public class TestRouterHDFSContractCreate extends AbstractContractCreateTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
RouterHDFSContract.createCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void teardownCluster() throws IOException {
|
||||||
|
RouterHDFSContract.destroyCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AbstractFSContract createContract(Configuration conf) {
|
||||||
|
return new RouterHDFSContract(conf);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,48 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test delete operations on the Router-based FS.
|
||||||
|
*/
|
||||||
|
public class TestRouterHDFSContractDelete extends AbstractContractDeleteTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
RouterHDFSContract.createCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void teardownCluster() throws IOException {
|
||||||
|
RouterHDFSContract.destroyCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AbstractFSContract createContract(Configuration conf) {
|
||||||
|
return new RouterHDFSContract(conf);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,49 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test get file status operations on the Router-based FS.
|
||||||
|
*/
|
||||||
|
public class TestRouterHDFSContractGetFileStatus
|
||||||
|
extends AbstractContractGetFileStatusTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
RouterHDFSContract.createCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void teardownCluster() throws IOException {
|
||||||
|
RouterHDFSContract.destroyCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AbstractFSContract createContract(Configuration conf) {
|
||||||
|
return new RouterHDFSContract(conf);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,48 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test dir operations on the Router-based FS.
|
||||||
|
*/
|
||||||
|
public class TestRouterHDFSContractMkdir extends AbstractContractMkdirTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
RouterHDFSContract.createCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void teardownCluster() throws IOException {
|
||||||
|
RouterHDFSContract.destroyCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AbstractFSContract createContract(Configuration conf) {
|
||||||
|
return new RouterHDFSContract(conf);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,48 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test open operations on the Router-based FS.
|
||||||
|
*/
|
||||||
|
public class TestRouterHDFSContractOpen extends AbstractContractOpenTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
RouterHDFSContract.createCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void teardownCluster() throws IOException {
|
||||||
|
RouterHDFSContract.destroyCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AbstractFSContract createContract(Configuration conf) {
|
||||||
|
return new RouterHDFSContract(conf);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,48 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test rename operations on the Router-based FS.
|
||||||
|
*/
|
||||||
|
public class TestRouterHDFSContractRename extends AbstractContractRenameTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
RouterHDFSContract.createCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void teardownCluster() throws IOException {
|
||||||
|
RouterHDFSContract.destroyCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AbstractFSContract createContract(Configuration conf) {
|
||||||
|
return new RouterHDFSContract(conf);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,64 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test root dir operations on the Router-based FS.
|
||||||
|
*/
|
||||||
|
public class TestRouterHDFSContractRootDirectory extends
|
||||||
|
AbstractContractRootDirectoryTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
RouterHDFSContract.createCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void teardownCluster() throws IOException {
|
||||||
|
RouterHDFSContract.destroyCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AbstractFSContract createContract(Configuration conf) {
|
||||||
|
return new RouterHDFSContract(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void testListEmptyRootDirectory() throws IOException {
|
||||||
|
// It doesn't apply because we still have the mount points here
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void testRmEmptyRootDirNonRecursive() throws IOException {
|
||||||
|
// It doesn't apply because we still have the mount points here
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void testRecursiveRootListing() throws IOException {
|
||||||
|
// It doesn't apply because we still have the mount points here
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,49 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test seek operations on the Router-based FS.
|
||||||
|
*/
|
||||||
|
public class TestRouterHDFSContractSeek extends AbstractContractSeekTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
RouterHDFSContract.createCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void teardownCluster() throws IOException {
|
||||||
|
RouterHDFSContract.destroyCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AbstractFSContract createContract(Configuration conf) {
|
||||||
|
return new RouterHDFSContract(conf);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,49 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.contract.router;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractContractSetTimesTest;
|
||||||
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test set times operations on the Router-based FS.
|
||||||
|
*/
|
||||||
|
public class TestRouterHDFSContractSetTimes
|
||||||
|
extends AbstractContractSetTimesTest {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void createCluster() throws IOException {
|
||||||
|
RouterHDFSContract.createCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void teardownCluster() throws IOException {
|
||||||
|
RouterHDFSContract.destroyCluster();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected AbstractFSContract createContract(Configuration conf) {
|
||||||
|
return new RouterHDFSContract(conf);
|
||||||
|
}
|
||||||
|
}
|
|
@ -147,6 +147,37 @@ public final class FederationTestUtils {
|
||||||
}, 1000, 20 * 1000);
|
}, 1000, 20 * 1000);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait for a namenode to be registered with a particular state.
|
||||||
|
* @param resolver Active namenode resolver.
|
||||||
|
* @param nsId Nameservice identifier.
|
||||||
|
* @param state State to check for.
|
||||||
|
* @throws Exception Failed to verify State Store registration of namenode
|
||||||
|
* nsId for state.
|
||||||
|
*/
|
||||||
|
public static void waitNamenodeRegistered(
|
||||||
|
final ActiveNamenodeResolver resolver, final String nsId,
|
||||||
|
final FederationNamenodeServiceState state) throws Exception {
|
||||||
|
|
||||||
|
GenericTestUtils.waitFor(new Supplier<Boolean>() {
|
||||||
|
@Override
|
||||||
|
public Boolean get() {
|
||||||
|
try {
|
||||||
|
List<? extends FederationNamenodeContext> nns =
|
||||||
|
resolver.getNamenodesForNameserviceId(nsId);
|
||||||
|
for (FederationNamenodeContext nn : nns) {
|
||||||
|
if (nn.getState().equals(state)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
// Ignore
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}, 1000, 20 * 1000);
|
||||||
|
}
|
||||||
|
|
||||||
public static boolean verifyDate(Date d1, Date d2, long precision) {
|
public static boolean verifyDate(Date d1, Date d2, long precision) {
|
||||||
return Math.abs(d1.getTime() - d2.getTime()) < precision;
|
return Math.abs(d1.getTime() - d2.getTime()) < precision;
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,6 +42,7 @@ import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_
|
||||||
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE;
|
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE;
|
||||||
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY;
|
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY;
|
||||||
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY;
|
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY;
|
||||||
|
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE;
|
||||||
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS;
|
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS;
|
||||||
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS;
|
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
|
@ -75,6 +76,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
|
||||||
import org.apache.hadoop.hdfs.MiniDFSNNTopology.NSConf;
|
import org.apache.hadoop.hdfs.MiniDFSNNTopology.NSConf;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
|
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
|
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
|
||||||
|
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
|
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
|
import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
|
||||||
import org.apache.hadoop.hdfs.server.federation.router.Router;
|
import org.apache.hadoop.hdfs.server.federation.router.Router;
|
||||||
|
@ -91,15 +93,16 @@ import org.slf4j.LoggerFactory;
|
||||||
/**
|
/**
|
||||||
* Test utility to mimic a federated HDFS cluster with multiple routers.
|
* Test utility to mimic a federated HDFS cluster with multiple routers.
|
||||||
*/
|
*/
|
||||||
public class RouterDFSCluster {
|
public class MiniRouterDFSCluster {
|
||||||
|
|
||||||
private static final Logger LOG =
|
private static final Logger LOG =
|
||||||
LoggerFactory.getLogger(RouterDFSCluster.class);
|
LoggerFactory.getLogger(MiniRouterDFSCluster.class);
|
||||||
|
|
||||||
public static final String TEST_STRING = "teststring";
|
public static final String TEST_STRING = "teststring";
|
||||||
public static final String TEST_DIR = "testdir";
|
public static final String TEST_DIR = "testdir";
|
||||||
public static final String TEST_FILE = "testfile";
|
public static final String TEST_FILE = "testfile";
|
||||||
|
|
||||||
|
private static final Random RND = new Random();
|
||||||
|
|
||||||
/** Nameservices in the federated cluster. */
|
/** Nameservices in the federated cluster. */
|
||||||
private List<String> nameservices;
|
private List<String> nameservices;
|
||||||
|
@ -345,7 +348,8 @@ public class RouterDFSCluster {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes,
|
public MiniRouterDFSCluster(
|
||||||
|
boolean ha, int numNameservices, int numNamenodes,
|
||||||
long heartbeatInterval, long cacheFlushInterval) {
|
long heartbeatInterval, long cacheFlushInterval) {
|
||||||
this.highAvailability = ha;
|
this.highAvailability = ha;
|
||||||
this.heartbeatInterval = heartbeatInterval;
|
this.heartbeatInterval = heartbeatInterval;
|
||||||
|
@ -353,12 +357,13 @@ public class RouterDFSCluster {
|
||||||
configureNameservices(numNameservices, numNamenodes);
|
configureNameservices(numNameservices, numNamenodes);
|
||||||
}
|
}
|
||||||
|
|
||||||
public RouterDFSCluster(boolean ha, int numNameservices) {
|
public MiniRouterDFSCluster(boolean ha, int numNameservices) {
|
||||||
this(ha, numNameservices, 2,
|
this(ha, numNameservices, 2,
|
||||||
DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
|
DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
|
||||||
}
|
}
|
||||||
|
|
||||||
public RouterDFSCluster(boolean ha, int numNameservices, int numNamenodes) {
|
public MiniRouterDFSCluster(
|
||||||
|
boolean ha, int numNameservices, int numNamenodes) {
|
||||||
this(ha, numNameservices, numNamenodes,
|
this(ha, numNameservices, numNamenodes,
|
||||||
DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
|
DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_CACHE_INTERVAL_MS);
|
||||||
}
|
}
|
||||||
|
@ -480,6 +485,9 @@ public class RouterDFSCluster {
|
||||||
conf.setClass(FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
|
conf.setClass(FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
|
||||||
MockResolver.class, FileSubclusterResolver.class);
|
MockResolver.class, FileSubclusterResolver.class);
|
||||||
|
|
||||||
|
// Disable safemode on startup
|
||||||
|
conf.setBoolean(DFS_ROUTER_SAFEMODE_ENABLE, false);
|
||||||
|
|
||||||
// Set the nameservice ID for the default NN monitor
|
// Set the nameservice ID for the default NN monitor
|
||||||
conf.set(DFS_NAMESERVICE_ID, nsId);
|
conf.set(DFS_NAMESERVICE_ID, nsId);
|
||||||
if (nnId != null) {
|
if (nnId != null) {
|
||||||
|
@ -549,8 +557,7 @@ public class RouterDFSCluster {
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getRandomNameservice() {
|
public String getRandomNameservice() {
|
||||||
Random r = new Random();
|
int randIndex = RND.nextInt(nameservices.size());
|
||||||
int randIndex = r.nextInt(nameservices.size());
|
|
||||||
return nameservices.get(randIndex);
|
return nameservices.get(randIndex);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -772,6 +779,22 @@ public class RouterDFSCluster {
|
||||||
waitNamenodeRegistered(nnResolver, nsId, nnId, state);
|
waitNamenodeRegistered(nnResolver, nsId, nnId, state);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait for name spaces to be active.
|
||||||
|
* @throws Exception If we cannot check the status or we timeout.
|
||||||
|
*/
|
||||||
|
public void waitActiveNamespaces() throws Exception {
|
||||||
|
for (RouterContext r : this.routers) {
|
||||||
|
Router router = r.router;
|
||||||
|
final ActiveNamenodeResolver resolver = router.getNamenodeResolver();
|
||||||
|
for (FederationNamespaceInfo ns : resolver.getNamespaces()) {
|
||||||
|
final String nsId = ns.getNameserviceId();
|
||||||
|
waitNamenodeRegistered(
|
||||||
|
resolver, nsId, FederationNamenodeServiceState.ACTIVE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the federated path for a nameservice.
|
* Get the federated path for a nameservice.
|
||||||
* @param nsId Nameservice identifier.
|
* @param nsId Nameservice identifier.
|
|
@ -42,7 +42,7 @@ import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
|
||||||
* Test utility to mimic a federated HDFS cluster with a router and a state
|
* Test utility to mimic a federated HDFS cluster with a router and a state
|
||||||
* store.
|
* store.
|
||||||
*/
|
*/
|
||||||
public class StateStoreDFSCluster extends RouterDFSCluster {
|
public class StateStoreDFSCluster extends MiniRouterDFSCluster {
|
||||||
|
|
||||||
private static final Class<?> DEFAULT_FILE_RESOLVER =
|
private static final Class<?> DEFAULT_FILE_RESOLVER =
|
||||||
MountTableResolver.class;
|
MountTableResolver.class;
|
||||||
|
|
|
@ -29,7 +29,7 @@ import java.util.List;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
|
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
|
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
|
||||||
|
|
|
@ -30,7 +30,7 @@ import java.util.List;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
|
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
|
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
|
||||||
|
|
|
@ -34,8 +34,8 @@ import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
|
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.MultipleDestinationMountTableResolver;
|
import org.apache.hadoop.hdfs.server.federation.resolver.MultipleDestinationMountTableResolver;
|
||||||
|
|
|
@ -28,8 +28,8 @@ import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
|
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
|
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
|
||||||
|
|
|
@ -27,8 +27,8 @@ import java.util.List;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hdfs.server.federation.MockResolver;
|
import org.apache.hadoop.hdfs.server.federation.MockResolver;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
|
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
|
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
|
||||||
import org.apache.hadoop.service.Service.STATE;
|
import org.apache.hadoop.service.Service.STATE;
|
||||||
|
@ -42,9 +42,9 @@ import org.junit.rules.TestName;
|
||||||
* Test the service that heartbeats the state of the namenodes to the State
|
* Test the service that heartbeats the state of the namenodes to the State
|
||||||
* Store.
|
* Store.
|
||||||
*/
|
*/
|
||||||
public class TestNamenodeHeartbeat {
|
public class TestRouterNamenodeHeartbeat {
|
||||||
|
|
||||||
private static RouterDFSCluster cluster;
|
private static MiniRouterDFSCluster cluster;
|
||||||
private static ActiveNamenodeResolver namenodeResolver;
|
private static ActiveNamenodeResolver namenodeResolver;
|
||||||
private static List<NamenodeHeartbeatService> services;
|
private static List<NamenodeHeartbeatService> services;
|
||||||
|
|
||||||
|
@ -54,7 +54,7 @@ public class TestNamenodeHeartbeat {
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void globalSetUp() throws Exception {
|
public static void globalSetUp() throws Exception {
|
||||||
|
|
||||||
cluster = new RouterDFSCluster(true, 2);
|
cluster = new MiniRouterDFSCluster(true, 2);
|
||||||
|
|
||||||
// Start NNs and DNs and wait until ready
|
// Start NNs and DNs and wait until ready
|
||||||
cluster.startCluster();
|
cluster.startCluster();
|
||||||
|
@ -91,7 +91,7 @@ public class TestNamenodeHeartbeat {
|
||||||
@Test
|
@Test
|
||||||
public void testNamenodeHeartbeatService() throws IOException {
|
public void testNamenodeHeartbeatService() throws IOException {
|
||||||
|
|
||||||
RouterDFSCluster testCluster = new RouterDFSCluster(true, 1);
|
MiniRouterDFSCluster testCluster = new MiniRouterDFSCluster(true, 1);
|
||||||
Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration(
|
Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration(
|
||||||
NAMESERVICES[0]);
|
NAMESERVICES[0]);
|
||||||
NamenodeHeartbeatService server = new NamenodeHeartbeatService(
|
NamenodeHeartbeatService server = new NamenodeHeartbeatService(
|
|
@ -28,8 +28,8 @@ import java.util.List;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
|
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
|
import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
|
||||||
|
|
|
@ -38,8 +38,8 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol;
|
||||||
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
|
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
|
||||||
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
|
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
|
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
|
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
|
||||||
|
|
|
@ -31,8 +31,8 @@ import org.apache.hadoop.hdfs.DFSClient;
|
||||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
|
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
|
import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
|
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
|
||||||
|
|
|
@ -23,7 +23,7 @@ import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.creat
|
||||||
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.deleteFile;
|
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.deleteFile;
|
||||||
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus;
|
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus;
|
||||||
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
|
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
|
||||||
import static org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.TEST_STRING;
|
import static org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.TEST_STRING;
|
||||||
import static org.junit.Assert.assertArrayEquals;
|
import static org.junit.Assert.assertArrayEquals;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertFalse;
|
import static org.junit.Assert.assertFalse;
|
||||||
|
@ -72,9 +72,9 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
||||||
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
|
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
|
||||||
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
|
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
|
||||||
import org.apache.hadoop.io.EnumSetWritable;
|
import org.apache.hadoop.io.EnumSetWritable;
|
||||||
|
@ -114,7 +114,7 @@ public class TestRouterRpc {
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Federated HDFS cluster. */
|
/** Federated HDFS cluster. */
|
||||||
private static RouterDFSCluster cluster;
|
private static MiniRouterDFSCluster cluster;
|
||||||
|
|
||||||
/** Random Router for this federated cluster. */
|
/** Random Router for this federated cluster. */
|
||||||
private RouterContext router;
|
private RouterContext router;
|
||||||
|
@ -142,7 +142,7 @@ public class TestRouterRpc {
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void globalSetUp() throws Exception {
|
public static void globalSetUp() throws Exception {
|
||||||
cluster = new RouterDFSCluster(false, 2);
|
cluster = new MiniRouterDFSCluster(false, 2);
|
||||||
// We need 6 DNs to test Erasure Coding with RS-6-3-64k
|
// We need 6 DNs to test Erasure Coding with RS-6-3-64k
|
||||||
cluster.setNumDatanodesPerNameservice(6);
|
cluster.setNumDatanodesPerNameservice(6);
|
||||||
|
|
||||||
|
@ -221,7 +221,7 @@ public class TestRouterRpc {
|
||||||
testRouter.close();
|
testRouter.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected RouterDFSCluster getCluster() {
|
protected MiniRouterDFSCluster getCluster() {
|
||||||
return TestRouterRpc.cluster;
|
return TestRouterRpc.cluster;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -38,9 +38,9 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol;
|
||||||
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
||||||
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
|
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
|
||||||
import org.apache.hadoop.hdfs.server.federation.MockResolver;
|
import org.apache.hadoop.hdfs.server.federation.MockResolver;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
|
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
|
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
|
import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
|
||||||
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
|
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
|
||||||
|
@ -54,7 +54,7 @@ public class TestRouterRpcMultiDestination extends TestRouterRpc {
|
||||||
@Override
|
@Override
|
||||||
public void testSetup() throws Exception {
|
public void testSetup() throws Exception {
|
||||||
|
|
||||||
RouterDFSCluster cluster = getCluster();
|
MiniRouterDFSCluster cluster = getCluster();
|
||||||
|
|
||||||
// Create mock locations
|
// Create mock locations
|
||||||
getCluster().installMockLocations();
|
getCluster().installMockLocations();
|
||||||
|
|
|
@ -0,0 +1,114 @@
|
||||||
|
<!--
|
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
~ or more contributor license agreements. See the NOTICE file
|
||||||
|
~ distributed with this work for additional information
|
||||||
|
~ regarding copyright ownership. The ASF licenses this file
|
||||||
|
~ to you under the Apache License, Version 2.0 (the
|
||||||
|
~ "License"); you may not use this file except in compliance
|
||||||
|
~ with the License. You may obtain a copy of the License at
|
||||||
|
~
|
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
~
|
||||||
|
~ Unless required by applicable law or agreed to in writing, software
|
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
~ See the License for the specific language governing permissions and
|
||||||
|
~ limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<configuration>
|
||||||
|
<!--
|
||||||
|
Here are most of the HDFS contract options.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.test.root-tests-enabled</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.file.contract.test.random-seek-count</name>
|
||||||
|
<value>500</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.is-case-sensitive</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-append</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-atomic-directory-delete</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-atomic-rename</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-block-locality</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-concat</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-seek</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.rejects-seek-past-eof</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-strict-exceptions</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-unix-permissions</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.rename-returns-false-if-dest-exists</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.rename-returns-false-if-source-missing</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-settimes</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-getfilestatus</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-file-reference</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.contract.supports-content-check</name>
|
||||||
|
<value>true</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
</configuration>
|
Loading…
Reference in New Issue