diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/PrivateAzureDataLakeFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/PrivateAzureDataLakeFileSystem.java index 89011d286ac..c4a19d5abdd 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/PrivateAzureDataLakeFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/hdfs/web/PrivateAzureDataLakeFileSystem.java @@ -922,7 +922,7 @@ public class PrivateAzureDataLakeFileSystem extends SWebHdfsFileSystem { size = maxBufferSize; } - int equalBufferSplit = Math.max(Math.round(size / SIZE4MB), 1); + int equalBufferSplit = Math.max(size / SIZE4MB, 1); int splitSize = Math.min(equalBufferSplit, maxConcurrentConnection); return splitSize; } diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestADLResponseData.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestADLResponseData.java new file mode 100644 index 00000000000..1c4fcabb21c --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestADLResponseData.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl; + +import org.apache.hadoop.fs.FileStatus; + +import java.util.Random; + +/** + * This class is responsible to provide generic test methods for mock up test + * to generate stub response for a network request. + */ +public final class TestADLResponseData { + + private TestADLResponseData() {} + + public static String getGetFileStatusJSONResponse(FileStatus status) { + String str = "{\"FileStatus\":{\"length\":" + status.getLen() + "," + + "\"pathSuffix\":\"\",\"type\":\"" + (status.isDirectory() ? + "DIRECTORY" : + "FILE") + "\"" + + ",\"blockSize\":" + status.getBlockSize() + ",\"accessTime\":" + + status.getAccessTime() + ",\"modificationTime\":" + status + .getModificationTime() + "" + + ",\"replication\":" + status.getReplication() + ",\"permission\":\"" + + status.getPermission() + "\",\"owner\":\"" + status.getOwner() + + "\",\"group\":\"" + status.getGroup() + "\"}}"; + + return str; + } + + public static String getGetFileStatusJSONResponse() { + return getGetFileStatusJSONResponse(4194304); + } + + public static String getGetFileStatusJSONResponse(long length) { + String str = "{\"FileStatus\":{\"length\":" + length + "," + + "\"pathSuffix\":\"\",\"type\":\"FILE\",\"blockSize\":268435456," + + "\"accessTime\":1452103827023,\"modificationTime\":1452103827023," + + "\"replication\":0,\"permission\":\"777\"," + + "\"owner\":\"NotSupportYet\",\"group\":\"NotSupportYet\"}}"; + return str; + } + + public static String getListFileStatusJSONResponse(int dirSize) { + String list = ""; + for (int i = 0; i < dirSize; ++i) { + list += "{\"length\":1024,\"pathSuffix\":\"" + java.util.UUID.randomUUID() + + "\",\"type\":\"FILE\",\"blockSize\":268435456," + + "\"accessTime\":1452103878833," + + "\"modificationTime\":1452103879190,\"replication\":0," + + "\"permission\":\"777\",\"owner\":\"NotSupportYet\"," + + "\"group\":\"NotSupportYet\"},"; + } + + list = list.substring(0, list.length() - 1); + String str = "{\"FileStatuses\":{\"FileStatus\":[" + list + "]}}"; + + return str; + } + + public static String getJSONResponse(boolean status) { + String str = "{\"boolean\":" + status + "}"; + return str; + } + + public static String getErrorIllegalArgumentExceptionJSONResponse() { + String str = "{\n" + + " \"RemoteException\":\n" + + " {\n" + + " \"exception\" : \"IllegalArgumentException\",\n" + + " \"javaClassName\": \"java.lang.IllegalArgumentException\",\n" + + " \"message\" : \"Bad Offset 0x83090015\"" + + " }\n" + + "}"; + + return str; + } + + public static String getErrorInternalServerExceptionJSONResponse() { + String str = "{\n" + + " \"RemoteException\":\n" + + " {\n" + + " \"exception\" : \"RumtimeException\",\n" + + " \"javaClassName\": \"java.lang.RumtimeException\",\n" + + " \"message\" : \"Internal Server Error\"" + + " }\n" + + "}"; + + return str; + } + + public static byte[] getRandomByteArrayData() { + return getRandomByteArrayData(4 * 1024 * 1024); + } + + public static byte[] getRandomByteArrayData(int size) { + byte[] b = new byte[size]; + Random rand = new Random(); + rand.nextBytes(b); + return b; + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java new file mode 100644 index 00000000000..4268b27f12c --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl; + +import com.squareup.okhttp.mockwebserver.MockResponse; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.common.AdlMockWebServer; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Time; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.net.URISyntaxException; + +/** + * This class is responsible for testing local getFileStatus implementation + * to cover correct parsing of successful and error JSON response + * from the server. + * Adls GetFileStatus operation is in detail covered in + * org.apache.hadoop.fs.adl.live testing package. + */ +public class TestGetFileStatus extends AdlMockWebServer { + + @Test + public void getFileStatusReturnsAsExpected() + throws URISyntaxException, IOException { + getMockServer().enqueue(new MockResponse().setResponseCode(200) + .setBody(TestADLResponseData.getGetFileStatusJSONResponse())); + long startTime = Time.monotonicNow(); + FileStatus fileStatus = getMockAdlFileSystem().getFileStatus( + new Path("/test1/test2")); + long endTime = Time.monotonicNow(); + System.out.println("Time : " + (endTime - startTime)); + Assert.assertTrue(fileStatus.isFile()); + Assert.assertEquals(fileStatus.getPath().toString(), + "adl://" + getMockServer().getHostName() + ":" + + getMockServer().getPort() + + "/test1/test2"); + Assert.assertEquals(fileStatus.getLen(), 4194304); + Assert.assertEquals(fileStatus.getBlockSize(), 268435456); + Assert.assertEquals(fileStatus.getReplication(), 0); + Assert.assertEquals(fileStatus.getPermission(), new FsPermission("777")); + Assert.assertEquals(fileStatus.getOwner(), "NotSupportYet"); + Assert.assertEquals(fileStatus.getGroup(), "NotSupportYet"); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java new file mode 100644 index 00000000000..82c24942e45 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl; + +import com.squareup.okhttp.mockwebserver.MockResponse; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.common.AdlMockWebServer; +import org.apache.hadoop.util.Time; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +/** + * This class is responsible for testing local listStatus implementation to + * cover correct parsing of successful and error JSON response from the server. + * Adls ListStatus functionality is in detail covered in + * org.apache.hadoop.fs.adl.live testing package. + */ +public class TestListStatus extends AdlMockWebServer { + + @Test + public void listStatusReturnsAsExpected() throws IOException { + getMockServer().enqueue(new MockResponse().setResponseCode(200) + .setBody(TestADLResponseData.getListFileStatusJSONResponse(10))); + long startTime = Time.monotonicNow(); + FileStatus[] ls = getMockAdlFileSystem().listStatus( + new Path("/test1/test2")); + long endTime = Time.monotonicNow(); + System.out.println("Time : " + (endTime - startTime)); + Assert.assertEquals(ls.length, 10); + + getMockServer().enqueue(new MockResponse().setResponseCode(200) + .setBody(TestADLResponseData.getListFileStatusJSONResponse(200))); + startTime = Time.monotonicNow(); + ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2")); + endTime = Time.monotonicNow(); + System.out.println("Time : " + (endTime - startTime)); + Assert.assertEquals(ls.length, 200); + + getMockServer().enqueue(new MockResponse().setResponseCode(200) + .setBody(TestADLResponseData.getListFileStatusJSONResponse(2048))); + startTime = Time.monotonicNow(); + ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2")); + endTime = Time.monotonicNow(); + System.out.println("Time : " + (endTime - startTime)); + Assert.assertEquals(ls.length, 2048); + } + + @Test + public void listStatusonFailure() throws IOException { + getMockServer().enqueue(new MockResponse().setResponseCode(403).setBody( + TestADLResponseData.getErrorIllegalArgumentExceptionJSONResponse())); + FileStatus[] ls = null; + long startTime = Time.monotonicNow(); + try { + ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2")); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains("Bad Offset 0x83090015")); + } + long endTime = Time.monotonicNow(); + System.out.println("Time : " + (endTime - startTime)); + + getMockServer().enqueue(new MockResponse().setResponseCode(500) + .setBody( + TestADLResponseData.getErrorInternalServerExceptionJSONResponse())); + startTime = Time.monotonicNow(); + try { + ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2")); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains("Internal Server Error")); + } + endTime = Time.monotonicNow(); + System.out.println("Time : " + (endTime - startTime)); + } + +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java new file mode 100644 index 00000000000..1e03bbf4528 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl; + +/** + * This class overrides AdlFileSystem to change transport scheme to http instead + * of https to run against Mock Server. + */ +public class TestableAdlFileSystem extends AdlFileSystem { + @Override + protected String getTransportScheme() { + return "http"; + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java new file mode 100644 index 00000000000..d257768da25 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.adl.AdlFileSystem; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * Utility class to configure real Adls storage to run Live test suite against. + */ +public final class AdlStorageConfiguration { + private AdlStorageConfiguration() {} + + private static final String CONTRACT_ENABLE_KEY = + "dfs.adl.test.contract" + ".enable"; + private static final String TEST_CONFIGURATION_FILE_NAME = + "contract-test-options.xml"; + private static final String TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME = + "adls.xml"; + + private static boolean isContractTestEnabled = false; + private static Configuration conf = null; + + public static Configuration getConfiguration() { + Configuration localConf = new Configuration(); + localConf.addResource(TEST_CONFIGURATION_FILE_NAME); + localConf.addResource(TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME); + return localConf; + } + + public static boolean isContractTestEnabled() { + if (conf == null) { + conf = getConfiguration(); + } + + isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY, false); + return isContractTestEnabled; + } + + public static FileSystem createAdlStorageConnector() + throws URISyntaxException, IOException { + if (conf == null) { + conf = getConfiguration(); + } + + if(!isContractTestEnabled()) { + return null; + } + + AdlFileSystem fileSystem = new AdlFileSystem(); + fileSystem.initialize(new URI(conf.get("fs.defaultFS")), conf); + return fileSystem; + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageContract.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageContract.java new file mode 100644 index 00000000000..1e401995506 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageContract.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +import java.io.IOException; +import java.net.URISyntaxException; + +/** + * Extension of AbstractFSContract representing a filesystem contract that + * a Adls filesystem implementation is expected implement. + */ +public class AdlStorageContract extends AbstractFSContract { + private FileSystem fs; + + protected AdlStorageContract(Configuration conf) { + super(conf); + try { + fs = AdlStorageConfiguration.createAdlStorageConnector(); + } catch (URISyntaxException e) { + throw new IllegalStateException("Can not initialize ADL FileSystem. " + + "Please check fs.defaultFS property.", e); + } catch (IOException e) { + throw new IllegalStateException("Can not initialize ADL FileSystem.", e); + } + this.setConf(AdlStorageConfiguration.getConfiguration()); + } + + @Override + public String getScheme() { + return "adl"; + } + + @Override + public FileSystem getTestFileSystem() throws IOException { + return this.fs; + } + + @Override + public Path getTestPath() { + Path path = new Path("/test"); + return path; + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractAppendLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractAppendLive.java new file mode 100644 index 00000000000..83390aff894 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractAppendLive.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractAppendTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Verify Adls APPEND semantics compliance with Hadoop. + */ +public class TestAdlContractAppendLive extends AbstractContractAppendTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Override + @Test + public void testRenameFileBeingAppended() throws Throwable { + ContractTestUtils.unsupported("Skipping since renaming file in append " + + "mode not supported in Adl"); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractConcatLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractConcatLive.java new file mode 100644 index 00000000000..8474e9cf274 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractConcatLive.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractConcatTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Verify Adls CONCAT semantics compliance with Hadoop. + */ +public class TestAdlContractConcatLive extends AbstractContractConcatTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } + + @Test + public void testConcatMissingTarget() throws Throwable { + ContractTestUtils.unsupported("BUG : Adl to support expectation from " + + "concat on missing targets."); + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractCreateLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractCreateLive.java new file mode 100644 index 00000000000..907c50c6812 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractCreateLive.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Verify Adls CREATE semantics compliance with Hadoop. + */ +public class TestAdlContractCreateLive extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } + + @Test + public void testOverwriteEmptyDirectory() throws Throwable { + ContractTestUtils + .unsupported("BUG : Adl to support override empty " + "directory."); + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractDeleteLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractDeleteLive.java new file mode 100644 index 00000000000..30eaec7fa4e --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractDeleteLive.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.Before; + +/** + * Verify Adls DELETE semantics compliance with Hadoop. + */ +public class TestAdlContractDeleteLive extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractMkdirLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractMkdirLive.java new file mode 100644 index 00000000000..e498110d88b --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractMkdirLive.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Verify Adls MKDIR semantics compliance with Hadoop. + */ +public class TestAdlContractMkdirLive extends AbstractContractMkdirTest { + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AdlStorageContract(conf); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } + + @Test + public void testMkdirOverParentFile() throws Throwable { + ContractTestUtils.unsupported("Not supported by Adl"); + } + + @Test + public void testNoMkdirOverFile() throws Throwable { + ContractTestUtils.unsupported("Not supported by Adl"); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractOpenLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractOpenLive.java new file mode 100644 index 00000000000..2bb2095e34e --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractOpenLive.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.Before; + +/** + * Verify Adls OPEN/READ semantics compliance with Hadoop. + */ +public class TestAdlContractOpenLive extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRenameLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRenameLive.java new file mode 100644 index 00000000000..06063c5a631 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRenameLive.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Verify Adls RENAME semantics compliance with Hadoop. + */ +public class TestAdlContractRenameLive extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } + + @Test + public void testRenameFileOverExistingFile() throws Throwable { + ContractTestUtils + .unsupported("BUG : Adl to support full complete POSIX" + "behaviour"); + } + + @Test + public void testRenameFileNonexistentDir() throws Throwable { + ContractTestUtils + .unsupported("BUG : Adl to support create dir is not " + "exist"); + } + + @Test + public void testRenameWithNonEmptySubDir() throws Throwable { + ContractTestUtils.unsupported("BUG : Adl to support non empty dir move."); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRootDirLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRootDirLive.java new file mode 100644 index 00000000000..bf4e549440d --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRootDirLive.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Verify Adls root level operation support. + */ +public class TestAdlContractRootDirLive + extends AbstractContractRootDirectoryTest { + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } + + @Test + public void testRmNonEmptyRootDirNonRecursive() throws Throwable { + ContractTestUtils.unsupported( + "BUG : Adl should throw exception instred " + "of returning false."); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractSeekLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractSeekLive.java new file mode 100644 index 00000000000..09764648a53 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractSeekLive.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.Before; + +/** + * Verify Adls OPEN/READ seek operation support. + */ +public class TestAdlContractSeekLive extends AbstractContractSeekTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java new file mode 100644 index 00000000000..b7129955455 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Random; + +/** + * Verify different data segment size writes ensure the integrity and + * order of the data. + */ +public class TestAdlDifferentSizeWritesLive { + + public static byte[] getRandomByteArrayData(int size) { + byte[] b = new byte[size]; + Random rand = new Random(); + rand.nextBytes(b); + return b; + } + + @Before + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + } + + @Test + public void testSmallDataWrites() throws IOException { + testDataIntegrity(4 * 1024 * 1024, 1 * 1024); + testDataIntegrity(4 * 1024 * 1024, 7 * 1024); + testDataIntegrity(4 * 1024 * 1024, 10); + testDataIntegrity(2 * 1024 * 1024, 10); + testDataIntegrity(1 * 1024 * 1024, 10); + testDataIntegrity(100, 1); + } + + @Test + public void testMediumDataWrites() throws IOException { + testDataIntegrity(4 * 1024 * 1024, 1 * 1024 * 1024); + testDataIntegrity(7 * 1024 * 1024, 2 * 1024 * 1024); + testDataIntegrity(9 * 1024 * 1024, 2 * 1024 * 1024); + testDataIntegrity(10 * 1024 * 1024, 3 * 1024 * 1024); + } + + private void testDataIntegrity(int totalSize, int chunkSize) + throws IOException { + Path path = new Path("/test/dataIntegrityCheck"); + FileSystem fs = null; + try { + fs = AdlStorageConfiguration.createAdlStorageConnector(); + } catch (URISyntaxException e) { + throw new IllegalStateException("Can not initialize ADL FileSystem. " + + "Please check fs.defaultFS property.", e); + } + byte[] expectedData = getRandomByteArrayData(totalSize); + + FSDataOutputStream out = fs.create(path, true); + int iteration = totalSize / chunkSize; + int reminderIteration = totalSize % chunkSize; + int offset = 0; + for (int i = 0; i < iteration; ++i) { + out.write(expectedData, offset, chunkSize); + offset += chunkSize; + } + + out.write(expectedData, offset, reminderIteration); + out.close(); + + byte[] actualData = new byte[totalSize]; + FSDataInputStream in = fs.open(path); + in.readFully(0, actualData); + in.close(); + Assert.assertArrayEquals(expectedData, actualData); + Assert.assertTrue(fs.delete(path, true)); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java new file mode 100644 index 00000000000..e007c4e7bca --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystemContractBaseTest; +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +import java.io.IOException; + +/** + * Verify Adls adhere to Hadoop file system semantics. + */ +public class TestAdlFileSystemContractLive extends FileSystemContractBaseTest { + private FileSystem adlStore; + + @Override + protected void setUp() throws Exception { + adlStore = AdlStorageConfiguration.createAdlStorageConnector(); + if (AdlStorageConfiguration.isContractTestEnabled()) { + fs = adlStore; + } + } + + @Override + protected void tearDown() throws Exception { + if (AdlStorageConfiguration.isContractTestEnabled()) { + cleanup(); + adlStore = null; + fs = null; + } + } + + private void cleanup() throws IOException { + adlStore.delete(new Path("/test"), true); + } + + @Override + protected void runTest() throws Throwable { + if (AdlStorageConfiguration.isContractTestEnabled()) { + super.runTest(); + } + } + + public void testGetFileStatus() throws IOException { + if (!AdlStorageConfiguration.isContractTestEnabled()) { + return; + } + + Path testPath = new Path("/test/adltest"); + if (adlStore.exists(testPath)) { + adlStore.delete(testPath, false); + } + + adlStore.create(testPath).close(); + assertTrue(adlStore.delete(testPath, false)); + } + + /** + * The following tests are failing on Azure Data Lake and the Azure Data Lake + * file system code needs to be modified to make them pass. + * A separate work item has been opened for this. + */ + @Test + @Override + public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception { + // BUG : Adl should return exception instead of false. + } + + @Test + @Override + public void testMkdirsWithUmask() throws Exception { + // Support under implementation in Adl + } + + @Test + @Override + public void testMoveFileUnderParent() throws Exception { + // BUG: Adl server should return expected status code. + } + + @Test + @Override + public void testRenameFileToSelf() throws Exception { + // BUG: Adl server should return expected status code. + } + + @Test + @Override + public void testRenameToDirWithSamePrefixAllowed() throws Exception { + // BUG: Adl server should return expected status code. + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlReadLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlReadLive.java new file mode 100644 index 00000000000..f1d909c88ec --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlReadLive.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Random; +import java.util.UUID; + +/** + * Verify different data segment size read from the file to ensure the + * integrity and order of the data over + * BufferManger and BatchByteArrayInputStream implementation. + */ +public class TestAdlReadLive { + private String expectedData = "1234567890abcdefghijklmnopqrstuvwxyz"; + + @Before + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + } + + private FileSystem getFileSystem() throws IOException, URISyntaxException { + return AdlStorageConfiguration.createAdlStorageConnector(); + } + + private void setupFile(Path path) throws IOException, URISyntaxException { + setupFile(path, expectedData); + } + + private void setupFile(Path path, String data) + throws IOException, URISyntaxException { + expectedData = data; + FileSystem fs = getFileSystem(); + fs.delete(path, true); + FSDataOutputStream fdis = fs.create(path); + fdis.writeBytes(expectedData); + fdis.close(); + fs.listStatus(path.getParent()); + long actualLen = fs.getFileStatus(path).getLen(); + long expectedLen = expectedData.length(); + System.out.println( + " Length of file : " + fs.getFileStatus(path).getLen() + " " + fs + .getUri()); + Assert.assertEquals(expectedLen, actualLen); + } + + @Test + public void + testOpenReadMoreThanAvailableBufferCrashFixIndexOutOfBoundsException() + throws Throwable { + Path path = new Path("/test1"); + FileSystem fs = getFileSystem(); + setupFile(path); + + if (fs.exists(path)) { + Assert.assertTrue(fs.delete(path, true)); + } + + FSDataOutputStream outputStream = fs.create(path); + final byte[] data = new byte[24 * 1024 * 1024]; + Random ran = new Random(); + ran.nextBytes(data); + outputStream.write(data); + + FSDataInputStream bb = fs.open(path); + byte[] expected = new byte[4 * 1024 * 1024]; + bb.read(); + bb.readFully(16711581, expected, 33, + 65640); // BugFix : Was causing crash IndexOutOfBoundsException + bb.seek(16711581); + bb.readFully(16711576, expected, 33, 65640); + bb.readFully(16711578, expected, 33, 65640); + bb.readFully(16711580, expected, 33, 65640); + bb.readFully(16711576, expected, 0, expected.length); + bb.seek(0); + expected = new byte[134144]; + while (bb.read() != -1){ + continue; + } + bb.readFully(0, data, 0, data.length); + } + + @Test + public void readNullData() throws IOException, URISyntaxException { + String data = "SPL \u0001Lorg.apache.hadoop.examples.terasort" + + ".TeraGen$RangeInputFormat$RangeInputSplit \u008DLK@Lorg.apache" + + ".hadoop.examples.terasort" + + ".TeraGen$RangeInputFormat$RangeInputSplit\u008DLK@\u008DLK@"; + Path path = new Path("/test4"); + FileSystem fs = this.getFileSystem(); + setupFile(path, data); + FSDataInputStream bb = fs.open(path); + int i = 0; + String actualData = new String(); + System.out.println("Data Length :" + expectedData.length()); + byte[] arr = new byte[data.length()]; + bb.readFully(0, arr); + actualData = new String(arr); + System.out.println(" Data : " + actualData); + Assert.assertEquals(actualData.length(), expectedData.length()); + + arr = new byte[data.length() - 7]; + bb.readFully(7, arr); + actualData = new String(arr); + Assert.assertEquals(actualData.length(), expectedData.length() - 7); + bb.close(); + } + + @Test + public void readTest() throws IOException, URISyntaxException { + Path path = new Path("/test4"); + FileSystem fs = this.getFileSystem(); + setupFile(path); + FSDataInputStream bb = fs.open(path); + int i = 0; + String actualData = new String(); + while (true) { + int c = bb.read(); + if (c < 0) { + break; + } + actualData += (char) c; + } + + byte[] b = new byte[100]; + System.out.println(bb.read(b, 9, 91)); + System.out.println(bb.read()); + System.out.println(bb.read()); + System.out.println(bb.read()); + System.out.println(bb.read()); + System.out.println(bb.read()); + System.out.println(bb.read()); + + bb.close(); + Assert.assertEquals(actualData, expectedData); + + for (int j = 0; j < 100; ++j) { + fs = this.getFileSystem(); + fs.exists(new Path("/test" + j)); + } + } + + @Test + public void readByteTest() throws IOException, URISyntaxException { + Path path = new Path("/test3"); + FileSystem fs = this.getFileSystem(); + setupFile(path); + FSDataInputStream bb = fs.open(path); + int i = 0; + byte[] data = new byte[expectedData.length()]; + int readByte = bb.read(data); + bb.close(); + Assert.assertEquals(readByte, expectedData.length()); + Assert.assertEquals(new String(data), expectedData); + } + + @Test + public void readByteFullyTest() throws IOException, URISyntaxException { + Path path = new Path("/test2"); + FileSystem fs = this.getFileSystem(); + setupFile(path); + FSDataInputStream bb = fs.open(path); + int i = 0; + byte[] data = new byte[expectedData.length()]; + bb.readFully(data); + bb.close(); + Assert.assertEquals(new String(data), expectedData); + + bb = fs.open(path); + bb.readFully(data, 0, data.length); + bb.close(); + Assert.assertEquals(new String(data), expectedData); + } + + @Test + public void readCombinationTest() throws IOException, URISyntaxException { + Path path = new Path("/test1"); + FileSystem fs = this.getFileSystem(); + setupFile(path); + FSDataInputStream bb = fs.open(path); + int i = 0; + byte[] data = new byte[5]; + int readByte = bb.read(data); + Assert.assertEquals(new String(data), expectedData.substring(0, 5)); + + bb.readFully(data, 0, data.length); + Assert.assertEquals(new String(data), expectedData.substring(5, 10)); + bb.close(); + bb = fs.open(path); + bb.readFully(5, data, 0, data.length); + Assert.assertEquals(new String(data), expectedData.substring(5, 10)); + + bb.read(data); + Assert.assertEquals(new String(data), expectedData.substring(0, 5)); + bb.close(); + bb = fs.open(path); + bb.read(new byte[100]); + bb.close(); + } + + @Test + public void readMultiSeekTest() throws IOException, URISyntaxException { + final Path path = new Path( + "/delete14/" + UUID.randomUUID().toString().replaceAll("-", "")); + FileSystem fs = this.getFileSystem(); + + final byte[] actualData = new byte[3267397]; + Random ran = new Random(); + ran.nextBytes(actualData); + byte[] testData = null; + + fs.delete(path, true); + FSDataOutputStream os = fs.create(path); + os.write(actualData); + os.close(); + + FSDataInputStream bb = fs.open(path); + byte[] data = new byte[16384]; + bb.readFully(3251013, data, 0, 16384); + testData = new byte[16384]; + System.arraycopy(actualData, 3251013, testData, 0, 16384); + Assert.assertArrayEquals(testData, data); + + data = new byte[1921]; + bb.readFully(3265476, data, 0, 1921); + testData = new byte[1921]; + System.arraycopy(actualData, 3265476, testData, 0, 1921); + Assert.assertArrayEquals(testData, data); + + data = new byte[3267394]; + bb.readFully(3, data, 0, 3267394); + testData = new byte[3267394]; + System.arraycopy(actualData, 3, testData, 0, 3267394); + Assert.assertArrayEquals(testData, data); + + data = new byte[3266943]; + bb.readFully(454, data, 0, 3266943); + testData = new byte[3266943]; + System.arraycopy(actualData, 454, testData, 0, 3266943); + Assert.assertArrayEquals(testData, data); + + data = new byte[3265320]; + bb.readFully(2077, data, 0, 3265320); + testData = new byte[3265320]; + System.arraycopy(actualData, 2077, testData, 0, 3265320); + Assert.assertArrayEquals(testData, data); + + bb.close(); + + bb = fs.open(path); + + data = new byte[3263262]; + bb.readFully(4135, data, 0, 3263262); + testData = new byte[3263262]; + System.arraycopy(actualData, 4135, testData, 0, 3263262); + Assert.assertArrayEquals(testData, data); + + data = new byte[2992591]; + bb.readFully(274806, data, 0, 2992591); + testData = new byte[2992591]; + System.arraycopy(actualData, 274806, testData, 0, 2992591); + Assert.assertArrayEquals(testData, data); + + data = new byte[1985665]; + bb.readFully(1281732, data, 0, 1985665); + testData = new byte[1985665]; + System.arraycopy(actualData, 1281732, testData, 0, 1985665); + Assert.assertArrayEquals(testData, data); + + data = new byte[3267394]; + try { + bb.readFully(2420207, data, 0, 3267394); + Assert.fail("EOF expected"); + } catch (IOException e) { + } + + bb.close(); + } + + @Test + public void allASCIICharTest() throws IOException, URISyntaxException { + final Path path = new Path( + "/delete14/" + UUID.randomUUID().toString().replaceAll("-", "")); + FileSystem fs = this.getFileSystem(); + final byte[] actualData = new byte[127]; + for (byte i = 0; i < 127; ++i) { + actualData[i] = i; + } + + fs.delete(path, true); + FSDataOutputStream os = fs.create(path); + os.write(actualData); + os.close(); + + FSDataInputStream bb = fs.open(path); + byte[] data = new byte[127]; + + bb.readFully(0, data, 0, data.length); + bb.close(); + Assert.assertArrayEquals(data, actualData); + + bb = fs.open(path); + int byteRead = 1; + while (bb.read() != -1) { + byteRead++; + } + + bb.seek(0); + byteRead = 1; + while (bb.read() != -1) { + byteRead++; + } + bb.close(); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextCreateMkdirLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextCreateMkdirLive.java new file mode 100644 index 00000000000..fc50c04b709 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextCreateMkdirLive.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.DelegateToFileSystem; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest; +import org.apache.hadoop.fs.FileContextTestHelper; +import org.apache.hadoop.fs.FileSystem; +import org.junit.Assume; +import org.junit.BeforeClass; + +import java.io.File; +import java.net.URI; + +/** + * Verify Adls file system adhere to Hadoop file system contract using bunch of + * available test in FileContextCreateMkdirBaseTest. + */ +public class TestAdlWebHdfsFileContextCreateMkdirLive + extends FileContextCreateMkdirBaseTest { + private static final String KEY_FILE_SYSTEM = "fs.defaultFS"; + + @Override + public void setUp() throws Exception { + Configuration conf = AdlStorageConfiguration.getConfiguration(); + String fileSystem = conf.get(KEY_FILE_SYSTEM); + if (fileSystem == null || fileSystem.trim().length() == 0) { + throw new Exception("Default file system not configured."); + } + URI uri = new URI(fileSystem); + FileSystem fs = AdlStorageConfiguration.createAdlStorageConnector(); + fc = FileContext.getFileContext( + new DelegateToFileSystem(uri, fs, conf, fs.getScheme(), false) { + }, conf); + super.setUp(); + } + + /** + * Required to override since the getRandmizedTestDir on Windows generates + * absolute path of the local file path which contains ":" character. + * Example file system path generated is "adl:///d:/a/b/c + * + * Adls does not support : character in the path hence overriding to remove + * unsupported character from the path. + * + * @return FileContextTestHelper + */ + @Override + protected FileContextTestHelper createFileContextHelper() { + return new FileContextTestHelper(new File( + RandomStringUtils.randomAlphanumeric(10)) + .getAbsolutePath().replaceAll(":", "")); + } + + @BeforeClass + public static void skipTestCheck() { + Assume.assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextMainOperationsLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextMainOperationsLive.java new file mode 100644 index 00000000000..b135550b5ad --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextMainOperationsLive.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.DelegateToFileSystem; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileContextMainOperationsBaseTest; +import org.apache.hadoop.fs.FileContextTestHelper; +import org.apache.hadoop.fs.FileSystem; +import org.junit.Assume; +import org.junit.BeforeClass; + +import java.io.File; +import java.io.IOException; +import java.net.URI; + +/** + * Verify Adls file system adhere to Hadoop file system contract using bunch of + * available test in FileContextMainOperationsBaseTest. + */ +public class TestAdlWebHdfsFileContextMainOperationsLive + extends FileContextMainOperationsBaseTest { + + private static final String KEY_FILE_SYSTEM = "fs.defaultFS"; + + @BeforeClass + public static void skipTestCheck() { + Assume.assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + } + + @Override + public void setUp() throws Exception { + Configuration conf = AdlStorageConfiguration.getConfiguration(); + String fileSystem = conf.get(KEY_FILE_SYSTEM); + if (fileSystem == null || fileSystem.trim().length() == 0) { + throw new Exception("Default file system not configured."); + } + URI uri = new URI(fileSystem); + FileSystem fs = AdlStorageConfiguration.createAdlStorageConnector(); + fc = FileContext.getFileContext( + new DelegateToFileSystem(uri, fs, conf, fs.getScheme(), false) { + }, conf); + super.setUp(); + } + + /** + * Required to override since the getRandmizedTestDir on Windows generates + * absolute path of the local file path which contains ":" character. + * Example file system path generated is "adl:///d:/a/b/c + * + * Adls does not support : character in the path hence overriding to remove + * unsupported character from the path. + * + * @return FileContextTestHelper + */ + @Override + protected FileContextTestHelper createFileContextHelper() { + return new FileContextTestHelper( + new File(RandomStringUtils.randomAlphanumeric(10)).getAbsolutePath() + .replaceAll(":", "")); + } + + @Override + protected boolean listCorruptedBlocksSupported() { + return false; + } + + @Override + public void testUnsupportedSymlink() throws IOException { + Assume.assumeTrue("Symbolic link are not supported by Adls", false); + } + + /** + * In case this test is causing failure due to + * java.lang.RuntimeException: java.io.FileNotFoundException: Hadoop bin + * directory does not exist: \hadoop-common-project + * \hadoop-common\target\bin -see https://wiki.apache + * .org/hadoop/WindowsProblems. then do build the hadoop dependencies + * otherwise mark this test as skip. + */ + @Override + public void testWorkingDirectory() throws Exception { + super.testWorkingDirectory(); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/oauth2/TestCachedRefreshTokenBasedAccessTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/oauth2/TestCachedRefreshTokenBasedAccessTokenProvider.java index e57d3a9498c..c0445948eb0 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/oauth2/TestCachedRefreshTokenBasedAccessTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/oauth2/TestCachedRefreshTokenBasedAccessTokenProvider.java @@ -20,17 +20,19 @@ package org.apache.hadoop.fs.adl.oauth2; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.web.oauth2.AccessTokenProvider; - import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY; import static org.apache.hadoop.hdfs.web.oauth2.ConfRefreshTokenBasedAccessTokenProvider.OAUTH_REFRESH_TOKEN_KEY; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * Verify cache behavior of ConfRefreshTokenBasedAccessTokenProvider instances. diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/AdlMockWebServer.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/AdlMockWebServer.java new file mode 100644 index 00000000000..69bb9aeafa4 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/AdlMockWebServer.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.common; + +import com.eclipsesource.json.JsonObject; +import com.squareup.okhttp.mockwebserver.MockResponse; +import com.squareup.okhttp.mockwebserver.MockWebServer; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.adl.TestableAdlFileSystem; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.web.oauth2.ConfCredentialBasedAccessTokenProvider; +import org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider; +import org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +/** + * Mock server to simulate Adls backend calls. This infrastructure is expandable + * to override expected server response based on the derived test functionality. + * Common functionality to generate token information before request is send to + * adls backend is also managed within AdlMockWebServer implementation. + */ +public class AdlMockWebServer { + + // Create a MockWebServer. These are lean enough that you can create a new + // instance for every unit test. + private MockWebServer server = null; + private TestableAdlFileSystem fs = null; + private int port = 0; + private Configuration conf = new Configuration(); + + public MockWebServer getMockServer() { + return server; + } + + public TestableAdlFileSystem getMockAdlFileSystem() { + return fs; + } + + public int getPort() { + return port; + } + + public Configuration getConf() { + return conf; + } + + public static MockResponse getTokenResponse() { + JsonObject jsonObject = new JsonObject() + .set(OAuth2Constants.EXPIRES_IN, "0987654321") + .set("token_type", "bearer").set(OAuth2Constants.ACCESS_TOKEN, "123"); + MockResponse oauth2Response = new MockResponse(); + oauth2Response.addHeader("Content-Type", "application/json"); + oauth2Response.setResponseCode(200); + oauth2Response.setBody(jsonObject.toString()); + return oauth2Response; + } + + @Before + public void preTestSetup() throws IOException, URISyntaxException { + + server = new MockWebServer(); + server.enqueue(getTokenResponse()); + + // Start the server. + server.start(); + + // Ask the server for its URL. You'll need this to make HTTP requests. + URL baseUrl = server.getUrl(""); + port = baseUrl.getPort(); + + // Exercise your application code, which should make those HTTP requests. + // Responses are returned in the same order that they are enqueued. + fs = new TestableAdlFileSystem(); + + conf.set(HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY, "MY_CLIENTID"); + conf.set(HdfsClientConfigKeys.ACCESS_TOKEN_PROVIDER_KEY, + ConfCredentialBasedAccessTokenProvider.class.getName()); + conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY, "true"); + conf.set(HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY, "http://localhost:" + + port + "/refresh"); + conf.set(CredentialBasedAccessTokenProvider.OAUTH_CREDENTIAL_KEY, + "credential"); + + URI uri = new URI("adl://localhost:" + port); + fs.initialize(uri, conf); + } + + @After + public void postTestSetup() throws IOException { + fs.close(); + server.shutdown(); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/ExpectedResponse.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/ExpectedResponse.java new file mode 100644 index 00000000000..727417e176f --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/ExpectedResponse.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.common; + + +import com.squareup.okhttp.mockwebserver.MockResponse; + +import java.util.ArrayList; + +/** + * Supporting class to hold expected MockResponse object along with parameters + * for validation in test methods. + */ +public class ExpectedResponse { + private MockResponse response; + private ArrayList expectedQueryParameters = new ArrayList(); + private int expectedBodySize; + private String httpRequestType; + + public int getExpectedBodySize() { + return expectedBodySize; + } + + public String getHttpRequestType() { + return httpRequestType; + } + + public ArrayList getExpectedQueryParameters() { + return expectedQueryParameters; + } + + public MockResponse getResponse() { + return response; + } + + ExpectedResponse set(MockResponse mockResponse) { + this.response = mockResponse; + return this; + } + + ExpectedResponse addExpectedQueryParam(String param) { + expectedQueryParameters.add(param); + return this; + } + + ExpectedResponse addExpectedBodySize(int bodySize) { + this.expectedBodySize = bodySize; + return this; + } + + ExpectedResponse addExpectedHttpRequestType(String expectedHttpRequestType) { + this.httpRequestType = expectedHttpRequestType; + return this; + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/TestDataForRead.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/TestDataForRead.java new file mode 100644 index 00000000000..150dc6cf0a9 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/TestDataForRead.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.common; + +import com.squareup.okhttp.mockwebserver.Dispatcher; +import com.squareup.okhttp.mockwebserver.MockResponse; +import com.squareup.okhttp.mockwebserver.RecordedRequest; +import okio.Buffer; +import org.apache.hadoop.fs.adl.TestADLResponseData; + +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Supporting class for mock test to validate Adls read operation using + * BufferManager.java and BatchByteArrayInputStream implementation. + */ +public class TestDataForRead { + + private byte[] actualData; + private ArrayList responses; + private Dispatcher dispatcher; + private int intensityOfTest; + private boolean checkOfNoOfCalls; + private int expectedNoNetworkCall; + + public TestDataForRead(final byte[] actualData, int expectedNoNetworkCall, + int intensityOfTest, boolean checkOfNoOfCalls) { + this.checkOfNoOfCalls = checkOfNoOfCalls; + this.actualData = actualData; + responses = new ArrayList(); + this.expectedNoNetworkCall = expectedNoNetworkCall; + this.intensityOfTest = intensityOfTest; + + dispatcher = new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest recordedRequest) + throws InterruptedException { + if (recordedRequest.getPath().equals("/refresh")) { + return AdlMockWebServer.getTokenResponse(); + } + + if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) { + return new MockResponse().setResponseCode(200).setBody( + TestADLResponseData + .getGetFileStatusJSONResponse(actualData.length)); + } + + if (recordedRequest.getRequestLine().contains("op=OPEN")) { + String request = recordedRequest.getRequestLine(); + int offset = 0; + int byteCount = 0; + + Pattern pattern = Pattern.compile("offset=([0-9]+)"); + Matcher matcher = pattern.matcher(request); + if (matcher.find()) { + System.out.println(matcher.group(1)); + offset = Integer.parseInt(matcher.group(1)); + } + + pattern = Pattern.compile("length=([0-9]+)"); + matcher = pattern.matcher(request); + if (matcher.find()) { + System.out.println(matcher.group(1)); + byteCount = Integer.parseInt(matcher.group(1)); + } + + Buffer buf = new Buffer(); + buf.write(actualData, offset, byteCount); + return new MockResponse().setResponseCode(200) + .setChunkedBody(buf, 4 * 1024 * 1024); + } + + return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501); + } + }; + } + + public boolean isCheckOfNoOfCalls() { + return checkOfNoOfCalls; + } + + public int getExpectedNoNetworkCall() { + return expectedNoNetworkCall; + } + + public int getIntensityOfTest() { + return intensityOfTest; + } + + public byte[] getActualData() { + return actualData; + } + + public ArrayList getResponses() { + return responses; + } + + public Dispatcher getDispatcher() { + return dispatcher; + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestAdlRead.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestAdlRead.java new file mode 100644 index 00000000000..60904d0c059 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestAdlRead.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.hdfs.web; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.adl.TestADLResponseData; +import org.apache.hadoop.fs.common.AdlMockWebServer; +import org.apache.hadoop.fs.common.TestDataForRead; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Random; + +/** + * This class is responsible for stress positional reads vs number of network + * calls required by to fetch the amount of data. Test does ensure the data + * integrity and order of the data is maintained. This tests are meant to test + * BufferManager.java and BatchByteArrayInputStream implementation. + */ +@RunWith(Parameterized.class) +public class TestAdlRead extends AdlMockWebServer { + + // Keeping timeout of 1 hour to ensure the test does complete and should + // not terminate due to high backend latency. + @Rule + public Timeout globalTimeout = new Timeout(60 * 60000); + private TestDataForRead testData; + + public TestAdlRead(TestDataForRead testData) { + this.testData = testData; + getConf().set("adl.feature.override.readahead.max.buffersize", "8192"); + getConf().set("adl.feature.override.readahead.max.concurrent.connection", + "1"); + } + + @Parameterized.Parameters(name = "{index}") + public static Collection testDataForReadOperation() { + return Arrays.asList(new Object[][] { + + //-------------------------- + // Test Data + //-------------------------- + {new TestDataForRead("Hello World".getBytes(), 3, 1000, true)}, + {new TestDataForRead( + ("the problem you appear to be wrestling with is that this doesn't " + + "display very well. ").getBytes(), 3, 1000, true)}, + {new TestDataForRead( + ("Chinese Indonesians (Indonesian: Orang Tionghoa-Indonesia; " + + "Chinese: " + + "trad ???????, simp ???????, pin Y�nd�n�x?y� Hu�r�n), are " + + "Indonesians descended from various Chinese ethnic groups, " + + "particularly Han.").getBytes(), 3, 1000, true)}, + {new TestDataForRead( + TestADLResponseData.getRandomByteArrayData(5 * 1024), 3, 1000, + true)}, {new TestDataForRead( + TestADLResponseData.getRandomByteArrayData(1 * 1024), 3, 50, true)}, + {new TestDataForRead( + TestADLResponseData.getRandomByteArrayData(8 * 1024), 3, 10, true)}, + {new TestDataForRead( + TestADLResponseData.getRandomByteArrayData(32 * 1024), 6, 10, + false)}, {new TestDataForRead( + TestADLResponseData.getRandomByteArrayData(48 * 1024), 8, 10, false)}}); + } + + @After + @Before + public void cleanReadBuffer() { + BufferManager.getInstance().clear(); + } + + @Test + public void testEntireBytes() throws IOException, InterruptedException { + getMockServer().setDispatcher(testData.getDispatcher()); + FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test")); + byte[] expectedData = new byte[testData.getActualData().length]; + Assert.assertEquals(in.read(expectedData), expectedData.length); + Assert.assertArrayEquals(expectedData, testData.getActualData()); + in.close(); + if (testData.isCheckOfNoOfCalls()) { + Assert.assertEquals(testData.getExpectedNoNetworkCall(), + getMockServer().getRequestCount()); + } + } + + @Test + public void testSeekOperation() throws IOException, InterruptedException { + getMockServer().setDispatcher(testData.getDispatcher()); + FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test")); + Random random = new Random(); + for (int i = 0; i < 1000; ++i) { + int position = random.nextInt(testData.getActualData().length); + in.seek(position); + Assert.assertEquals(in.getPos(), position); + Assert.assertEquals(in.read(), testData.getActualData()[position] & 0xFF); + } + in.close(); + if (testData.isCheckOfNoOfCalls()) { + Assert.assertEquals(testData.getExpectedNoNetworkCall(), + getMockServer().getRequestCount()); + } + } + + @Test + public void testReadServerCalls() throws IOException, InterruptedException { + getMockServer().setDispatcher(testData.getDispatcher()); + FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test")); + byte[] expectedData = new byte[testData.getActualData().length]; + in.readFully(expectedData); + Assert.assertArrayEquals(expectedData, testData.getActualData()); + Assert.assertEquals(testData.getExpectedNoNetworkCall(), + getMockServer().getRequestCount()); + in.close(); + } + + @Test + public void testReadFully() throws IOException, InterruptedException { + getMockServer().setDispatcher(testData.getDispatcher()); + FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test")); + byte[] expectedData = new byte[testData.getActualData().length]; + in.readFully(expectedData); + Assert.assertArrayEquals(expectedData, testData.getActualData()); + + in.readFully(0, expectedData); + Assert.assertArrayEquals(expectedData, testData.getActualData()); + + in.seek(0); + in.readFully(expectedData, 0, expectedData.length); + Assert.assertArrayEquals(expectedData, testData.getActualData()); + in.close(); + + if (testData.isCheckOfNoOfCalls()) { + Assert.assertEquals(testData.getExpectedNoNetworkCall(), + getMockServer().getRequestCount()); + } + } + + @Test + public void testRandomPositionalReadUsingReadFully() + throws IOException, InterruptedException { + getMockServer().setDispatcher(testData.getDispatcher()); + FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test")); + ByteArrayInputStream actualData = new ByteArrayInputStream( + testData.getActualData()); + Random random = new Random(); + for (int i = 0; i < testData.getIntensityOfTest(); ++i) { + int offset = random.nextInt(testData.getActualData().length); + int length = testData.getActualData().length - offset; + byte[] expectedData = new byte[length]; + byte[] actualDataSubset = new byte[length]; + actualData.reset(); + actualData.skip(offset); + actualData.read(actualDataSubset, 0, length); + + in.readFully(offset, expectedData, 0, length); + Assert.assertArrayEquals(expectedData, actualDataSubset); + } + + for (int i = 0; i < testData.getIntensityOfTest(); ++i) { + int offset = random.nextInt(testData.getActualData().length); + int length = random.nextInt(testData.getActualData().length - offset); + byte[] expectedData = new byte[length]; + byte[] actualDataSubset = new byte[length]; + actualData.reset(); + actualData.skip(offset); + actualData.read(actualDataSubset, 0, length); + + in.readFully(offset, expectedData, 0, length); + Assert.assertArrayEquals(expectedData, actualDataSubset); + } + + in.close(); + if (testData.isCheckOfNoOfCalls()) { + Assert.assertEquals(testData.getExpectedNoNetworkCall(), + getMockServer().getRequestCount()); + } + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConcurrentDataReadOperations.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConcurrentDataReadOperations.java new file mode 100644 index 00000000000..1194336c1a2 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConcurrentDataReadOperations.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.hdfs.web; + +import com.squareup.okhttp.mockwebserver.Dispatcher; +import com.squareup.okhttp.mockwebserver.MockResponse; +import com.squareup.okhttp.mockwebserver.RecordedRequest; +import okio.Buffer; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.adl.TestADLResponseData; +import org.apache.hadoop.fs.common.AdlMockWebServer; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This class is responsible for testing multiple threads trying to access same + * or multiple files from the offset. This tests are meant to test + * BufferManager.java and BatchByteArrayInputStream implementation. + */ +@RunWith(Parameterized.class) +public class TestConcurrentDataReadOperations extends AdlMockWebServer { + + private static FSDataInputStream commonHandle = null; + private static Object lock = new Object(); + private int concurrencyLevel; + + public TestConcurrentDataReadOperations(int concurrencyLevel) { + this.concurrencyLevel = concurrencyLevel; + getConf().set("adl.feature.override.readahead.max.buffersize", "102400"); + getConf().set("adl.feature.override.readahead.max.concurrent.connection", + "1"); + } + + @Parameterized.Parameters(name = "{index}") + public static Collection testDataNumberOfConcurrentRun() { + return Arrays.asList(new Object[][] {{1}, {2}, {3}, {4}, {5}}); + } + + public static byte[] getRandomByteArrayData(int size) { + byte[] b = new byte[size]; + Random rand = new Random(); + rand.nextBytes(b); + return b; + } + + private void setDispatcher(final ArrayList testData) { + getMockServer().setDispatcher(new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest recordedRequest) + throws InterruptedException { + if (recordedRequest.getPath().equals("/refresh")) { + return AdlMockWebServer.getTokenResponse(); + } + + CreateTestData currentRequest = null; + for (CreateTestData local : testData) { + if (recordedRequest.getPath().contains(local.path.toString())) { + currentRequest = local; + break; + } + } + + if (currentRequest == null) { + new MockResponse().setBody("Request data not found") + .setResponseCode(501); + } + + if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) { + return new MockResponse().setResponseCode(200).setBody( + TestADLResponseData + .getGetFileStatusJSONResponse(currentRequest.data.length)); + } + + if (recordedRequest.getRequestLine().contains("op=OPEN")) { + String request = recordedRequest.getRequestLine(); + int offset = 0; + int byteCount = 0; + + Pattern pattern = Pattern.compile("offset=([0-9]+)"); + Matcher matcher = pattern.matcher(request); + if (matcher.find()) { + System.out.println(matcher.group(1)); + offset = Integer.parseInt(matcher.group(1)); + } + + pattern = Pattern.compile("length=([0-9]+)"); + matcher = pattern.matcher(request); + if (matcher.find()) { + System.out.println(matcher.group(1)); + byteCount = Integer.parseInt(matcher.group(1)); + } + + Buffer buf = new Buffer(); + buf.write(currentRequest.data, offset, byteCount); + return new MockResponse().setResponseCode(200) + .setChunkedBody(buf, 4 * 1024 * 1024); + } + + return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501); + } + }); + } + + @Before + public void resetHandle() { + commonHandle = null; + } + + @Test + public void testParallelReadOnDifferentStreams() + throws IOException, InterruptedException, ExecutionException { + + ArrayList createTestData = new ArrayList(); + + Random random = new Random(); + + for (int i = 0; i < concurrencyLevel; i++) { + CreateTestData testData = new CreateTestData(); + testData + .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()), + getRandomByteArrayData(random.nextInt(1 * 1024 * 1024))); + createTestData.add(testData); + } + + setDispatcher(createTestData); + + ArrayList readTestData = new ArrayList(); + for (CreateTestData local : createTestData) { + ReadTestData localReadData = new ReadTestData(); + localReadData.set(local.path, local.data, 0); + readTestData.add(localReadData); + } + + runReadTest(readTestData, false); + } + + @Test + public void testParallelReadOnSameStreams() + throws IOException, InterruptedException, ExecutionException { + ArrayList createTestData = new ArrayList(); + + Random random = new Random(); + + for (int i = 0; i < 1; i++) { + CreateTestData testData = new CreateTestData(); + testData + .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()), + getRandomByteArrayData(1024 * 1024)); + createTestData.add(testData); + } + + setDispatcher(createTestData); + + ArrayList readTestData = new ArrayList(); + ByteArrayInputStream buffered = new ByteArrayInputStream( + createTestData.get(0).data); + + ReadTestData readInitially = new ReadTestData(); + byte[] initialData = new byte[1024 * 1024]; + buffered.read(initialData); + + readInitially.set(createTestData.get(0).path, initialData, 0); + readTestData.add(readInitially); + runReadTest(readTestData, false); + + readTestData.clear(); + + for (int i = 0; i < concurrencyLevel * 5; i++) { + ReadTestData localReadData = new ReadTestData(); + int offset = random.nextInt((1024 * 1024)-1); + int length = 1024 * 1024 - offset; + byte[] expectedData = new byte[length]; + buffered.reset(); + buffered.skip(offset); + buffered.read(expectedData); + localReadData.set(createTestData.get(0).path, expectedData, offset); + readTestData.add(localReadData); + } + + runReadTest(readTestData, true); + } + + void runReadTest(ArrayList testData, boolean useSameStream) + throws InterruptedException, ExecutionException { + + ExecutorService executor = Executors.newFixedThreadPool(testData.size()); + Future[] subtasks = new Future[testData.size()]; + + for (int i = 0; i < testData.size(); i++) { + subtasks[i] = executor.submit( + new ReadConcurrentRunnable(testData.get(i).data, testData.get(i).path, + testData.get(i).offset, useSameStream)); + } + + executor.shutdown(); + + // wait until all tasks are finished + executor.awaitTermination(120, TimeUnit.SECONDS); + + for (int i = 0; i < testData.size(); ++i) { + Assert.assertTrue((Boolean) subtasks[i].get()); + } + } + + class ReadTestData { + private Path path; + private byte[] data; + private int offset; + + public void set(Path filePath, byte[] dataToBeRead, int fromOffset) { + this.path = filePath; + this.data = dataToBeRead; + this.offset = fromOffset; + } + } + + class CreateTestData { + private Path path; + private byte[] data; + + public void set(Path filePath, byte[] dataToBeWritten) { + this.path = filePath; + this.data = dataToBeWritten; + } + } + + class ReadConcurrentRunnable implements Callable { + private Path path; + private int offset; + private byte[] expectedData; + private boolean useSameStream; + + public ReadConcurrentRunnable(byte[] expectedData, Path path, int offset, + boolean useSameStream) { + this.path = path; + this.offset = offset; + this.expectedData = expectedData; + this.useSameStream = useSameStream; + } + + public Boolean call() throws IOException { + try { + FSDataInputStream in; + if (useSameStream) { + synchronized (lock) { + if (commonHandle == null) { + commonHandle = getMockAdlFileSystem().open(path); + } + in = commonHandle; + } + } else { + in = getMockAdlFileSystem().open(path); + } + + byte[] actualData = new byte[expectedData.length]; + in.readFully(offset, actualData); + Assert.assertArrayEquals("Path :" + path.toString() + " did not match.", + expectedData, actualData); + if (!useSameStream) { + in.close(); + } + } catch (IOException e) { + e.printStackTrace(); + return false; + } + return true; + } + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java new file mode 100644 index 00000000000..9a54ec25aa3 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.hdfs.web; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.adl.TestableAdlFileSystem; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.web.oauth2.ConfCredentialBasedAccessTokenProvider; +import org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * This class is responsible for testing adl file system required configuration + * and feature set keys. + */ +public class TestConfigurationSetting { + + @Test + public void testAllConfiguration() throws URISyntaxException, IOException { + TestableAdlFileSystem fs = new TestableAdlFileSystem(); + Configuration conf = new Configuration(); + conf.set(HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY, + "http://localhost:1111/refresh"); + conf.set(CredentialBasedAccessTokenProvider.OAUTH_CREDENTIAL_KEY, + "credential"); + conf.set(HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY, "MY_CLIENTID"); + conf.set(HdfsClientConfigKeys.ACCESS_TOKEN_PROVIDER_KEY, + ConfCredentialBasedAccessTokenProvider.class.getName()); + conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY, "true"); + + URI uri = new URI("adl://localhost:1234"); + fs.initialize(uri, conf); + + // Default setting check + Assert.assertEquals(true, fs.isFeatureRedirectOff()); + Assert.assertEquals(true, fs.isFeatureGetBlockLocationLocallyBundled()); + Assert.assertEquals(true, fs.isFeatureConcurrentReadWithReadAhead()); + Assert.assertEquals(false, fs.isOverrideOwnerFeatureOn()); + Assert.assertEquals(8 * 1024 * 1024, fs.getMaxBufferSize()); + Assert.assertEquals(2, fs.getMaxConcurrentConnection()); + + fs.close(); + + // Configuration toggle check + conf.set("adl.feature.override.redirection.off", "false"); + fs.initialize(uri, conf); + Assert.assertEquals(false, fs.isFeatureRedirectOff()); + fs.close(); + conf.set("adl.feature.override.redirection.off", "true"); + fs.initialize(uri, conf); + Assert.assertEquals(true, fs.isFeatureRedirectOff()); + fs.close(); + + conf.set("adl.feature.override.getblocklocation.locally.bundled", "false"); + fs.initialize(uri, conf); + Assert.assertEquals(false, fs.isFeatureGetBlockLocationLocallyBundled()); + fs.close(); + conf.set("adl.feature.override.getblocklocation.locally.bundled", "true"); + fs.initialize(uri, conf); + Assert.assertEquals(true, fs.isFeatureGetBlockLocationLocallyBundled()); + fs.close(); + + conf.set("adl.feature.override.readahead", "false"); + fs.initialize(uri, conf); + Assert.assertEquals(false, fs.isFeatureConcurrentReadWithReadAhead()); + fs.close(); + conf.set("adl.feature.override.readahead", "true"); + fs.initialize(uri, conf); + Assert.assertEquals(true, fs.isFeatureConcurrentReadWithReadAhead()); + fs.close(); + + conf.set("adl.feature.override.readahead.max.buffersize", "101"); + fs.initialize(uri, conf); + Assert.assertEquals(101, fs.getMaxBufferSize()); + fs.close(); + conf.set("adl.feature.override.readahead.max.buffersize", "12134565"); + fs.initialize(uri, conf); + Assert.assertEquals(12134565, fs.getMaxBufferSize()); + fs.close(); + + conf.set("adl.debug.override.localuserasfileowner", "true"); + fs.initialize(uri, conf); + Assert.assertEquals(true, fs.isOverrideOwnerFeatureOn()); + fs.close(); + conf.set("adl.debug.override.localuserasfileowner", "false"); + fs.initialize(uri, conf); + Assert.assertEquals(false, fs.isOverrideOwnerFeatureOn()); + fs.close(); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestSplitSizeCalculation.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestSplitSizeCalculation.java new file mode 100644 index 00000000000..42a4701d8ad --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestSplitSizeCalculation.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.hdfs.web; + +import com.squareup.okhttp.mockwebserver.MockResponse; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.adl.TestADLResponseData; +import org.apache.hadoop.fs.common.AdlMockWebServer; +import org.apache.hadoop.hdfs.web.PrivateAzureDataLakeFileSystem.BatchByteArrayInputStream; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; + +/** + * This class is responsible for testing split size calculation during + * read ahead buffer initiation based on the data size and configuration + * initialization. + */ +public class TestSplitSizeCalculation extends AdlMockWebServer { + + @Test + public void testSplitSizeCalculations() + throws URISyntaxException, IOException { + + getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody( + TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024))); + getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody( + TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024))); + getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody( + TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024))); + getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody( + TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024))); + + URL url = getMockServer().getUrl(""); + + BatchByteArrayInputStream stream = getMockAdlFileSystem() + .new BatchByteArrayInputStream(url, + new Path("/test1/test2"), 16 * 1024 * 1024, 4); + Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024)); + Assert.assertEquals(2, stream.getSplitSize(8 * 1024 * 1024)); + Assert.assertEquals(4, stream.getSplitSize(16 * 1024 * 1024)); + Assert.assertEquals(3, stream.getSplitSize(12 * 1024 * 1024)); + Assert.assertEquals(4, stream.getSplitSize(102 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(102)); + stream.close(); + + stream = getMockAdlFileSystem().new BatchByteArrayInputStream(url, + new Path("/test1/test2"), 4 * 1024 * 1024, 4); + Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(8 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(16 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(12 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(102 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(102)); + stream.close(); + + stream = getMockAdlFileSystem().new BatchByteArrayInputStream(url, + new Path("/test1/test2"), 16 * 1024 * 1024, 2); + Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024)); + Assert.assertEquals(2, stream.getSplitSize(8 * 1024 * 1024)); + Assert.assertEquals(2, stream.getSplitSize(16 * 1024 * 1024)); + Assert.assertEquals(2, stream.getSplitSize(12 * 1024 * 1024)); + Assert.assertEquals(2, stream.getSplitSize(102 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(102)); + stream.close(); + + stream = getMockAdlFileSystem().new BatchByteArrayInputStream(url, + new Path("/test1/test2"), 8 * 1024 * 1024, 2); + Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024)); + Assert.assertEquals(2, stream.getSplitSize(8 * 1024 * 1024)); + Assert.assertEquals(2, stream.getSplitSize(16 * 1024 * 1024)); + Assert.assertEquals(2, stream.getSplitSize(12 * 1024 * 1024)); + Assert.assertEquals(2, stream.getSplitSize(102 * 1024 * 1024)); + Assert.assertEquals(1, stream.getSplitSize(102)); + stream.close(); + } +} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml new file mode 100644 index 00000000000..f72de165705 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml @@ -0,0 +1,139 @@ + + + + fs.contract.test.root-tests-enabled + true + + + + fs.contract.test.supports-concat + true + + + + fs.contract.rename-returns-false-if-source-missing + true + + + + fs.contract.test.random-seek-count + 10 + + + + fs.contract.is-case-sensitive + true + + + + fs.contract.rename-returns-true-if-dest-exists + true + + + + fs.contract.rename-returns-true-if-source-missing + true + + + + fs.contract.rename-creates-dest-dirs + true + + + + fs.contract.rename-remove-dest-if-empty-dir + true + + + + fs.contract.supports-settimes + true + + + + fs.contract.supports-append + true + + + + fs.contract.supports-atomic-directory-delete + true + + + + fs.contract.supports-atomic-rename + true + + + + fs.contract.supports-block-locality + true + + + + fs.contract.supports-concat + true + + + + fs.contract.supports-seek + true + + + + fs.contract.supports-seek-on-closed-file + true + + + + fs.contract.rejects-seek-past-eof + true + + + + fs.contract.supports-available-on-closed-file + true + + + + fs.contract.supports-strict-exceptions + false + + + + fs.contract.supports-unix-permissions + true + + + + fs.contract.rename-overwrites-dest + true + + + + fs.contract.supports-append + true + + + + fs.azure.enable.append.support + true + + + + fs.contract.supports-getfilestatus + true + + diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml new file mode 100644 index 00000000000..160fba41267 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml @@ -0,0 +1,57 @@ + + + + dfs.webhdfs.oauth2.refresh.token.expires.ms.since.epoch + 0 + + + dfs.webhdfs.oauth2.credential + bearer.and.refresh.token + + + dfs.webhdfs.oauth2.refresh.url + https://login.windows.net/common/oauth2/token/ + + + dfs.webhdfs.oauth2.access.token.provider + + org.apache.hadoop.fs.adl.oauth2.CachedRefreshTokenBasedAccessTokenProvider + + + + dfs.webhdfs.oauth2.enabled + true + + + + dfs.webhdfs.oauth2.client.id + ADD CLIENT ID + + + + dfs.webhdfs.oauth2.refresh.token + ADD REFRESH TOKEN + + + + fs.defaultFS + adl://urAdlAccountGoesHere.azuredatalakestore.net:443/ + + + + dfs.adl.test.contract.enable + false + + \ No newline at end of file