From 4392325546a49d29720fe705381447a0fdd5826e Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Tue, 5 May 2015 11:54:30 +0530 Subject: [PATCH] HDFS-8242. Erasure Coding: XML based end-to-end test for ECCli commands (Contributed by Rakesh R) --- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 + .../hdfs/tools/erasurecode/ECCommand.java | 9 +- .../hadoop/cli/CLITestCmdErasureCoding.java | 38 ++ .../hadoop/cli/TestErasureCodingCLI.java | 114 ++++++ .../cli/util/CLICommandErasureCodingCli.java | 21 ++ .../cli/util/ErasureCodingCliCmdExecutor.java | 37 ++ .../test/resources/testErasureCodingConf.xml | 342 ++++++++++++++++++ 7 files changed, 561 insertions(+), 3 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandErasureCodingCli.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index faec023add4..ef760fcd9c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -166,3 +166,6 @@ (jing9) HDFS-8137. Send the EC schema to DataNode via EC encoding/recovering command(umamahesh) + + HDFS-8242. Erasure Coding: XML based end-to-end test for ECCli commands + (Rakesh R via vinayakumarb) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java index 84c22751a38..802a46dd08c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java @@ -17,7 +17,9 @@ package org.apache.hadoop.hdfs.tools.erasurecode; import java.io.IOException; +import java.util.ArrayList; import java.util.LinkedList; +import java.util.List; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; @@ -120,11 +122,12 @@ public abstract class ECCommand extends Command { sb.append("Schema '"); sb.append(schemaName); sb.append("' does not match any of the supported schemas."); - sb.append("Please select any one of ["); + sb.append(" Please select any one of "); + List schemaNames = new ArrayList(); for (ECSchema ecSchema : ecSchemas) { - sb.append(ecSchema.getSchemaName()); - sb.append(", "); + schemaNames.add(ecSchema.getSchemaName()); } + sb.append(schemaNames); throw new HadoopIllegalArgumentException(sb.toString()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java new file mode 100644 index 00000000000..6c06a8da85a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.cli; + +import org.apache.hadoop.cli.util.CLICommandErasureCodingCli; +import org.apache.hadoop.cli.util.CLICommandTypes; +import org.apache.hadoop.cli.util.CLITestCmd; +import org.apache.hadoop.cli.util.CommandExecutor; +import org.apache.hadoop.cli.util.ErasureCodingCliCmdExecutor; +import org.apache.hadoop.hdfs.tools.erasurecode.ECCli; + +public class CLITestCmdErasureCoding extends CLITestCmd { + public CLITestCmdErasureCoding(String str, CLICommandTypes type) { + super(str, type); + } + + @Override + public CommandExecutor getExecutor(String tag) throws IllegalArgumentException { + if (getType() instanceof CLICommandErasureCodingCli) + return new ErasureCodingCliCmdExecutor(tag, new ECCli()); + return super.getExecutor(tag); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java new file mode 100644 index 00000000000..5f01ea29164 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.cli; + +import org.apache.hadoop.cli.util.CLICommand; +import org.apache.hadoop.cli.util.CLICommandErasureCodingCli; +import org.apache.hadoop.cli.util.CommandExecutor.Result; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.xml.sax.SAXException; + +public class TestErasureCodingCLI extends CLITestHelper { + private final int NUM_OF_DATANODES = 3; + private MiniDFSCluster dfsCluster = null; + private FileSystem fs = null; + private String namenode = null; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + + dfsCluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(NUM_OF_DATANODES).build(); + dfsCluster.waitClusterUp(); + namenode = conf.get(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "file:///"); + + username = System.getProperty("user.name"); + + fs = dfsCluster.getFileSystem(); + } + + @Override + protected String getTestFile() { + return "testErasureCodingConf.xml"; + } + + @After + @Override + public void tearDown() throws Exception { + if (fs != null) { + fs.close(); + } + if (dfsCluster != null) { + dfsCluster.shutdown(); + } + Thread.sleep(2000); + super.tearDown(); + } + + @Override + protected String expandCommand(final String cmd) { + String expCmd = cmd; + expCmd = expCmd.replaceAll("NAMENODE", namenode); + expCmd = expCmd.replaceAll("#LF#", System.getProperty("line.separator")); + expCmd = super.expandCommand(expCmd); + return expCmd; + } + + @Override + protected TestConfigFileParser getConfigParser() { + return new TestErasureCodingAdmin(); + } + + private class TestErasureCodingAdmin extends + CLITestHelper.TestConfigFileParser { + @Override + public void endElement(String uri, String localName, String qName) + throws SAXException { + if (qName.equals("ec-admin-command")) { + if (testCommands != null) { + testCommands.add(new CLITestCmdErasureCoding(charString, + new CLICommandErasureCodingCli())); + } else if (cleanupCommands != null) { + cleanupCommands.add(new CLITestCmdErasureCoding(charString, + new CLICommandErasureCodingCli())); + } + } else { + super.endElement(uri, localName, qName); + } + } + } + + @Override + protected Result execute(CLICommand cmd) throws Exception { + return cmd.getExecutor(namenode).executeCommand(cmd.getCmd()); + } + + @Test + @Override + public void testAll() { + super.testAll(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandErasureCodingCli.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandErasureCodingCli.java new file mode 100644 index 00000000000..aafcd9fa774 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandErasureCodingCli.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.cli.util; + +public class CLICommandErasureCodingCli implements CLICommandTypes { +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java new file mode 100644 index 00000000000..e993313a8fb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.cli.util; + +import org.apache.hadoop.hdfs.tools.erasurecode.ECCli; +import org.apache.hadoop.util.ToolRunner; + +public class ErasureCodingCliCmdExecutor extends CommandExecutor { + protected String namenode = null; + protected ECCli admin = null; + + public ErasureCodingCliCmdExecutor(String namenode, ECCli admin) { + this.namenode = namenode; + this.admin = admin; + } + + @Override + protected void execute(final String cmd) throws Exception { + String[] args = getCommandAsArgs(cmd, "NAMENODE", this.namenode); + ToolRunner.run(admin, args); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml new file mode 100644 index 00000000000..b7b29d3a696 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -0,0 +1,342 @@ + + + + + + + + test + + + + + + + help: help for erasure coding command + + -help + + + + + + SubstringComparator + Usage: hdfs erasurecode [generic options] + + + + + + help: createZone command + + -fs NAMENODE -help createZone + + + + + + RegexpComparator + ^[ \t]*Create a zone to encode files using a specified schema( )* + + + RegexpComparator + ^-createZone \[-s <schemaName>\] <path>(.)* + + + + + + help: getZoneInfo command + + -fs NAMENODE -help getZoneInfo + + + + + + SubstringComparator + Get information about the EC zone at specified path + + + RegexpComparator + ^-getZoneInfo <path>(.)* + + + + + + help: listSchemas command + + -fs NAMENODE -help listSchemas + + + + + + SubstringComparator + Get the list of ECSchemas supported + + + RegexpComparator + ^-listSchemas (.)* + + + + + + + createZone : create a zone to encode files + + -fs NAMENODE -mkdir /eczone + -fs NAMENODE -createZone -s RS-6-3 /eczone + + + -fs NAMENODE -rmdir /eczone + + + + SubstringComparator + EC Zone created successfully at NAMENODE/eczone + + + + + + createZone : default schema + + -fs NAMENODE -mkdir /eczone + -fs NAMENODE -createZone /eczone + -fs NAMENODE -getZoneInfo /eczone + + + -fs NAMENODE -rmdir /eczone + + + + SubstringComparator + Dir: /eczone, Schema: ECSchema=[Name=RS-6-3 + + + + + + getZoneInfo : get information about the EC zone at specified path + + -fs NAMENODE -mkdir /eczone + -fs NAMENODE -createZone -s RS-6-3 /eczone + -fs NAMENODE -getZoneInfo /eczone + + + -fs NAMENODE -rmdir /eczone + + + + SubstringComparator + Dir: /eczone, Schema: ECSchema=[Name=RS-6-3 + + + + + + getZoneInfo : get EC zone at specified file path + + -fs NAMENODE -mkdir /eczone + -fs NAMENODE -createZone -s RS-6-3 /eczone + -fs NAMENODE -touchz /eczone/ecfile + -fs NAMENODE -getZoneInfo /eczone/ecfile + + + -fs NAMENODE -rm /eczone/ecfile + -fs NAMENODE -rmdir /eczone + + + + SubstringComparator + Dir: /eczone, Schema: ECSchema=[Name=RS-6-3 + + + + + + listSchemas : get the list of ECSchemas supported + + -fs NAMENODE -listSchemas + + + + + + SubstringComparator + RS-6-3 + + + + + + + createZone : illegal parameters - path is missing + + -fs NAMENODE -mkdir /eczone + -fs NAMENODE -createZone + + + -fs NAMENODE -rmdir /eczone + + + + RegexpComparator + ^-createZone: <path> is missing(.)* + + + + + + createZone : illegal parameters - schema name is missing + + -fs NAMENODE -mkdir /eczone + -fs NAMENODE -createZone -s + + + -fs NAMENODE -rmdir /eczone + + + + RegexpComparator + ^-createZone: option -s requires 1 argument(.)* + + + + + + createZone : illegal parameters - too many arguments + + -fs NAMENODE -mkdir /eczone + -fs NAMENODE -createZone /eczone1 /eczone2 + + + -fs NAMENODE -rmdir /eczone + + + + SubstringComparator + -createZone: Too many arguments + + + + + + createZone : illegal parameters - invalidschema + + -fs NAMENODE -mkdir /eczone + -fs NAMENODE -createZone -s invalidschema /eczone + + + -fs NAMENODE -rmdir /eczone + + + + SubstringComparator + Schema 'invalidschema' does not match any of the supported schemas. Please select any one of [RS-6-3] + + + + + + createZone : illegal parameters - no such file + + -fs NAMENODE -createZone /eczone + + + + + + RegexpComparator + ^createZone: `/eczone': No such file or directory(.)* + + + + + + getZoneInfo : illegal parameters - path is missing + + -fs NAMENODE -getZoneInfo + + + + + + RegexpComparator + ^-getZoneInfo: <path> is missing(.)* + + + + + + getZoneInfo : illegal parameters - too many arguments + + -fs NAMENODE -getZoneInfo /eczone /eczone + + + -fs NAMENODE -rm /eczone + + + + SubstringComparator + -getZoneInfo: Too many arguments + + + + + + getZoneInfo : illegal parameters - no such file + + -fs NAMENODE -getZoneInfo /eczone + + + + + + RegexpComparator + ^getZoneInfo: `/eczone': No such file or directory(.)* + + + + + + listSchemas : illegal parameters - too many parameters + + -fs NAMENODE -listSchemas /eczone + + + + + + SubstringComparator + -listSchemas: Too many parameters + + + + + +