HDFS-8242. Erasure Coding: XML based end-to-end test for ECCli commands (Contributed by Rakesh R)

This commit is contained in:
Vinayakumar B 2015-05-05 11:54:30 +05:30 committed by Zhe Zhang
parent 436c14855a
commit 4392325546
7 changed files with 561 additions and 3 deletions

View File

@ -166,3 +166,6 @@
(jing9) (jing9)
HDFS-8137. Send the EC schema to DataNode via EC encoding/recovering command(umamahesh) HDFS-8137. Send the EC schema to DataNode via EC encoding/recovering command(umamahesh)
HDFS-8242. Erasure Coding: XML based end-to-end test for ECCli commands
(Rakesh R via vinayakumarb)

View File

@ -17,7 +17,9 @@
package org.apache.hadoop.hdfs.tools.erasurecode; package org.apache.hadoop.hdfs.tools.erasurecode;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List;
import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
@ -120,11 +122,12 @@ public abstract class ECCommand extends Command {
sb.append("Schema '"); sb.append("Schema '");
sb.append(schemaName); sb.append(schemaName);
sb.append("' does not match any of the supported schemas."); sb.append("' does not match any of the supported schemas.");
sb.append("Please select any one of ["); sb.append(" Please select any one of ");
List<String> schemaNames = new ArrayList<String>();
for (ECSchema ecSchema : ecSchemas) { for (ECSchema ecSchema : ecSchemas) {
sb.append(ecSchema.getSchemaName()); schemaNames.add(ecSchema.getSchemaName());
sb.append(", ");
} }
sb.append(schemaNames);
throw new HadoopIllegalArgumentException(sb.toString()); throw new HadoopIllegalArgumentException(sb.toString());
} }
} }

View File

@ -0,0 +1,38 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.cli;
import org.apache.hadoop.cli.util.CLICommandErasureCodingCli;
import org.apache.hadoop.cli.util.CLICommandTypes;
import org.apache.hadoop.cli.util.CLITestCmd;
import org.apache.hadoop.cli.util.CommandExecutor;
import org.apache.hadoop.cli.util.ErasureCodingCliCmdExecutor;
import org.apache.hadoop.hdfs.tools.erasurecode.ECCli;
public class CLITestCmdErasureCoding extends CLITestCmd {
public CLITestCmdErasureCoding(String str, CLICommandTypes type) {
super(str, type);
}
@Override
public CommandExecutor getExecutor(String tag) throws IllegalArgumentException {
if (getType() instanceof CLICommandErasureCodingCli)
return new ErasureCodingCliCmdExecutor(tag, new ECCli());
return super.getExecutor(tag);
}
}

View File

@ -0,0 +1,114 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.cli;
import org.apache.hadoop.cli.util.CLICommand;
import org.apache.hadoop.cli.util.CLICommandErasureCodingCli;
import org.apache.hadoop.cli.util.CommandExecutor.Result;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.xml.sax.SAXException;
public class TestErasureCodingCLI extends CLITestHelper {
private final int NUM_OF_DATANODES = 3;
private MiniDFSCluster dfsCluster = null;
private FileSystem fs = null;
private String namenode = null;
@Before
@Override
public void setUp() throws Exception {
super.setUp();
dfsCluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_OF_DATANODES).build();
dfsCluster.waitClusterUp();
namenode = conf.get(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "file:///");
username = System.getProperty("user.name");
fs = dfsCluster.getFileSystem();
}
@Override
protected String getTestFile() {
return "testErasureCodingConf.xml";
}
@After
@Override
public void tearDown() throws Exception {
if (fs != null) {
fs.close();
}
if (dfsCluster != null) {
dfsCluster.shutdown();
}
Thread.sleep(2000);
super.tearDown();
}
@Override
protected String expandCommand(final String cmd) {
String expCmd = cmd;
expCmd = expCmd.replaceAll("NAMENODE", namenode);
expCmd = expCmd.replaceAll("#LF#", System.getProperty("line.separator"));
expCmd = super.expandCommand(expCmd);
return expCmd;
}
@Override
protected TestConfigFileParser getConfigParser() {
return new TestErasureCodingAdmin();
}
private class TestErasureCodingAdmin extends
CLITestHelper.TestConfigFileParser {
@Override
public void endElement(String uri, String localName, String qName)
throws SAXException {
if (qName.equals("ec-admin-command")) {
if (testCommands != null) {
testCommands.add(new CLITestCmdErasureCoding(charString,
new CLICommandErasureCodingCli()));
} else if (cleanupCommands != null) {
cleanupCommands.add(new CLITestCmdErasureCoding(charString,
new CLICommandErasureCodingCli()));
}
} else {
super.endElement(uri, localName, qName);
}
}
}
@Override
protected Result execute(CLICommand cmd) throws Exception {
return cmd.getExecutor(namenode).executeCommand(cmd.getCmd());
}
@Test
@Override
public void testAll() {
super.testAll();
}
}

View File

@ -0,0 +1,21 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.cli.util;
public class CLICommandErasureCodingCli implements CLICommandTypes {
}

View File

@ -0,0 +1,37 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.cli.util;
import org.apache.hadoop.hdfs.tools.erasurecode.ECCli;
import org.apache.hadoop.util.ToolRunner;
public class ErasureCodingCliCmdExecutor extends CommandExecutor {
protected String namenode = null;
protected ECCli admin = null;
public ErasureCodingCliCmdExecutor(String namenode, ECCli admin) {
this.namenode = namenode;
this.admin = admin;
}
@Override
protected void execute(final String cmd) throws Exception {
String[] args = getCommandAsArgs(cmd, "NAMENODE", this.namenode);
ToolRunner.run(admin, args);
}
}

View File

@ -0,0 +1,342 @@
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="testConf.xsl"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<!-- Normal mode is test. To run just the commands and dump the output
to the log, set it to nocompare -->
<mode>test</mode>
<!-- Comparator types:
ExactComparator
SubstringComparator
RegexpComparator
TokenComparator
-->
<tests>
<!-- Test help options -->
<test>
<description>help: help for erasure coding command</description>
<test-commands>
<ec-admin-command>-help</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>Usage: hdfs erasurecode [generic options]</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>help: createZone command</description>
<test-commands>
<ec-admin-command>-fs NAMENODE -help createZone</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^[ \t]*Create a zone to encode files using a specified schema( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-createZone \[-s &lt;schemaName&gt;\] &lt;path&gt;(.)*</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>help: getZoneInfo command</description>
<test-commands>
<ec-admin-command>-fs NAMENODE -help getZoneInfo</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>Get information about the EC zone at specified path</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-getZoneInfo &lt;path&gt;(.)*</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>help: listSchemas command</description>
<test-commands>
<ec-admin-command>-fs NAMENODE -help listSchemas</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>Get the list of ECSchemas supported</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-listSchemas (.)*</expected-output>
</comparator>
</comparators>
</test>
<!-- Test erasure code commands -->
<test>
<description>createZone : create a zone to encode files</description>
<test-commands>
<command>-fs NAMENODE -mkdir /eczone</command>
<ec-admin-command>-fs NAMENODE -createZone -s RS-6-3 /eczone</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /eczone</command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>EC Zone created successfully at NAMENODE/eczone</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>createZone : default schema</description>
<test-commands>
<command>-fs NAMENODE -mkdir /eczone</command>
<ec-admin-command>-fs NAMENODE -createZone /eczone</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getZoneInfo /eczone</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /eczone</command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>Dir: /eczone, Schema: ECSchema=[Name=RS-6-3</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>getZoneInfo : get information about the EC zone at specified path</description>
<test-commands>
<command>-fs NAMENODE -mkdir /eczone</command>
<ec-admin-command>-fs NAMENODE -createZone -s RS-6-3 /eczone</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getZoneInfo /eczone</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /eczone</command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>Dir: /eczone, Schema: ECSchema=[Name=RS-6-3</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>getZoneInfo : get EC zone at specified file path</description>
<test-commands>
<command>-fs NAMENODE -mkdir /eczone</command>
<ec-admin-command>-fs NAMENODE -createZone -s RS-6-3 /eczone</ec-admin-command>
<command>-fs NAMENODE -touchz /eczone/ecfile</command>
<ec-admin-command>-fs NAMENODE -getZoneInfo /eczone/ecfile</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /eczone/ecfile</command>
<command>-fs NAMENODE -rmdir /eczone</command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>Dir: /eczone, Schema: ECSchema=[Name=RS-6-3</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>listSchemas : get the list of ECSchemas supported</description>
<test-commands>
<ec-admin-command>-fs NAMENODE -listSchemas</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>RS-6-3</expected-output>
</comparator>
</comparators>
</test>
<!-- Test illegal parameters -->
<test>
<description>createZone : illegal parameters - path is missing</description>
<test-commands>
<command>-fs NAMENODE -mkdir /eczone</command>
<ec-admin-command>-fs NAMENODE -createZone</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /eczone</command>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-createZone: &lt;path&gt; is missing(.)*</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>createZone : illegal parameters - schema name is missing</description>
<test-commands>
<command>-fs NAMENODE -mkdir /eczone</command>
<ec-admin-command>-fs NAMENODE -createZone -s</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /eczone</command>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-createZone: option -s requires 1 argument(.)*</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>createZone : illegal parameters - too many arguments</description>
<test-commands>
<command>-fs NAMENODE -mkdir /eczone</command>
<ec-admin-command>-fs NAMENODE -createZone /eczone1 /eczone2</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /eczone</command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>-createZone: Too many arguments</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>createZone : illegal parameters - invalidschema</description>
<test-commands>
<command>-fs NAMENODE -mkdir /eczone</command>
<ec-admin-command>-fs NAMENODE -createZone -s invalidschema /eczone</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /eczone</command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>Schema 'invalidschema' does not match any of the supported schemas. Please select any one of [RS-6-3]</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>createZone : illegal parameters - no such file</description>
<test-commands>
<ec-admin-command>-fs NAMENODE -createZone /eczone</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^createZone: `/eczone': No such file or directory(.)*</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>getZoneInfo : illegal parameters - path is missing</description>
<test-commands>
<ec-admin-command>-fs NAMENODE -getZoneInfo </ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-getZoneInfo: &lt;path&gt; is missing(.)*</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>getZoneInfo : illegal parameters - too many arguments</description>
<test-commands>
<ec-admin-command>-fs NAMENODE -getZoneInfo /eczone /eczone</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /eczone</command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>-getZoneInfo: Too many arguments</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>getZoneInfo : illegal parameters - no such file</description>
<test-commands>
<ec-admin-command>-fs NAMENODE -getZoneInfo /eczone</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^getZoneInfo: `/eczone': No such file or directory(.)*</expected-output>
</comparator>
</comparators>
</test>
<test>
<description>listSchemas : illegal parameters - too many parameters</description>
<test-commands>
<ec-admin-command>-fs NAMENODE -listSchemas /eczone</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>-listSchemas: Too many parameters</expected-output>
</comparator>
</comparators>
</test>
</tests>
</configuration>