From bd5ab5912564d2d687651b01f552b8e4ca8c145a Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 12 Feb 2018 15:11:17 +0000 Subject: [PATCH] HADOOP-15187. Remove ADL mock test dependency on REST call invoked from Java SDK. Contributed by Vishwajeet Dusane. --- .../hadoop/fs/adl/AdlMockWebServer.java | 102 ------ .../apache/hadoop/fs/adl/TestACLFeatures.java | 262 --------------- .../org/apache/hadoop/fs/adl/TestAdlRead.java | 196 ------------ .../adl/TestConcurrentDataReadOperations.java | 299 ------------------ .../fs/adl/TestCustomTokenProvider.java | 140 -------- .../hadoop/fs/adl/TestGetFileStatus.java | 102 ------ .../apache/hadoop/fs/adl/TestListStatus.java | 137 -------- .../hadoop/fs/adl/TestableAdlFileSystem.java | 30 -- .../fs/adl/common/ExpectedResponse.java | 71 ----- .../hadoop/fs/adl/common/TestDataForRead.java | 122 ------- 10 files changed, 1461 deletions(-) delete mode 100644 hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java delete mode 100644 hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestACLFeatures.java delete mode 100644 hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAdlRead.java delete mode 100644 hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestConcurrentDataReadOperations.java delete mode 100644 hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java delete mode 100644 hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java delete mode 100644 hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java delete mode 100644 hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java delete mode 100644 hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/ExpectedResponse.java delete mode 100644 hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/TestDataForRead.java diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java deleted file mode 100644 index d843d5556b3..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.adl; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider; -import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider; -import static org.apache.hadoop.fs.adl.AdlConfKeys - .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; -import static org.apache.hadoop.fs.adl.AdlConfKeys - .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; - -import com.squareup.okhttp.mockwebserver.MockWebServer; - -import org.junit.After; -import org.junit.Before; - -/** - * Mock server to simulate Adls backend calls. This infrastructure is expandable - * to override expected server response based on the derived test functionality. - * Common functionality to generate token information before request is send to - * adls backend is also managed within AdlMockWebServer implementation using - * {@link org.apache.hadoop.fs.adl.common.CustomMockTokenProvider}. - */ -public class AdlMockWebServer { - // Create a MockWebServer. These are lean enough that you can create a new - // instance for every unit test. - private MockWebServer server = null; - private TestableAdlFileSystem fs = null; - private int port = 0; - private Configuration conf = new Configuration(); - - public MockWebServer getMockServer() { - return server; - } - - public TestableAdlFileSystem getMockAdlFileSystem() { - return fs; - } - - public int getPort() { - return port; - } - - public Configuration getConf() { - return conf; - } - - public void setConf(Configuration conf) { - this.conf = conf; - } - - @Before - public void preTestSetup() throws IOException, URISyntaxException { - server = new MockWebServer(); - - // Start the server. - server.start(); - - // Ask the server for its URL. You'll need this to make HTTP requests. - URL baseUrl = server.getUrl(""); - port = baseUrl.getPort(); - - // Exercise your application code, which should make those HTTP requests. - // Responses are returned in the same order that they are enqueued. - fs = new TestableAdlFileSystem(); - - conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); - conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, - CustomMockTokenProvider.class, AzureADTokenProvider.class); - - URI uri = new URI("adl://localhost:" + port); - fs.initialize(uri, conf); - } - - @After - public void postTestSetup() throws IOException { - fs.close(); - server.shutdown(); - } -} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestACLFeatures.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestACLFeatures.java deleted file mode 100644 index b420daae337..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestACLFeatures.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.fs.adl; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.AclEntry; -import org.apache.hadoop.fs.permission.AclEntryScope; -import org.apache.hadoop.fs.permission.AclEntryType; -import org.apache.hadoop.fs.permission.AclStatus; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.security.AccessControlException; - -import com.squareup.okhttp.mockwebserver.MockResponse; - -import org.junit.Assert; -import org.junit.Test; - -/** - * Stub adl server and test acl data conversion within SDK and Hadoop adl - * client. - */ -public class TestACLFeatures extends AdlMockWebServer { - - @Test(expected=AccessControlException.class) - public void testModifyAclEntries() throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - List entries = new ArrayList(); - AclEntry.Builder aclEntryBuilder = new AclEntry.Builder(); - aclEntryBuilder.setName("hadoop"); - aclEntryBuilder.setType(AclEntryType.USER); - aclEntryBuilder.setPermission(FsAction.ALL); - aclEntryBuilder.setScope(AclEntryScope.ACCESS); - entries.add(aclEntryBuilder.build()); - - aclEntryBuilder.setName("hdfs"); - aclEntryBuilder.setType(AclEntryType.GROUP); - aclEntryBuilder.setPermission(FsAction.READ_WRITE); - aclEntryBuilder.setScope(AclEntryScope.DEFAULT); - entries.add(aclEntryBuilder.build()); - - getMockAdlFileSystem().modifyAclEntries(new Path("/test1/test2"), entries); - - getMockServer().enqueue(new MockResponse().setResponseCode(403) - .setBody(TestADLResponseData.getAccessControlException())); - - getMockAdlFileSystem() - .modifyAclEntries(new Path("/test1/test2"), entries); - } - - @Test(expected=AccessControlException.class) - public void testRemoveAclEntriesWithOnlyUsers() - throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - List entries = new ArrayList(); - AclEntry.Builder aclEntryBuilder = new AclEntry.Builder(); - aclEntryBuilder.setName("hadoop"); - aclEntryBuilder.setType(AclEntryType.USER); - entries.add(aclEntryBuilder.build()); - - getMockAdlFileSystem().removeAclEntries(new Path("/test1/test2"), entries); - - getMockServer().enqueue(new MockResponse().setResponseCode(403) - .setBody(TestADLResponseData.getAccessControlException())); - - getMockAdlFileSystem() - .removeAclEntries(new Path("/test1/test2"), entries); - } - - @Test(expected=AccessControlException.class) - public void testRemoveAclEntries() throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - List entries = new ArrayList(); - AclEntry.Builder aclEntryBuilder = new AclEntry.Builder(); - aclEntryBuilder.setName("hadoop"); - aclEntryBuilder.setType(AclEntryType.USER); - aclEntryBuilder.setPermission(FsAction.ALL); - aclEntryBuilder.setScope(AclEntryScope.ACCESS); - entries.add(aclEntryBuilder.build()); - - aclEntryBuilder.setName("hdfs"); - aclEntryBuilder.setType(AclEntryType.GROUP); - aclEntryBuilder.setPermission(FsAction.READ_WRITE); - aclEntryBuilder.setScope(AclEntryScope.DEFAULT); - entries.add(aclEntryBuilder.build()); - - getMockAdlFileSystem().removeAclEntries(new Path("/test1/test2"), entries); - - getMockServer().enqueue(new MockResponse().setResponseCode(403) - .setBody(TestADLResponseData.getAccessControlException())); - - getMockAdlFileSystem() - .removeAclEntries(new Path("/test1/test2"), entries); - } - - @Test(expected=AccessControlException.class) - public void testRemoveDefaultAclEntries() - throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem().removeDefaultAcl(new Path("/test1/test2")); - - getMockServer().enqueue(new MockResponse().setResponseCode(403) - .setBody(TestADLResponseData.getAccessControlException())); - - getMockAdlFileSystem().removeDefaultAcl(new Path("/test1/test2")); - } - - @Test(expected=AccessControlException.class) - public void testRemoveAcl() throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem().removeAcl(new Path("/test1/test2")); - - getMockServer().enqueue(new MockResponse().setResponseCode(403) - .setBody(TestADLResponseData.getAccessControlException())); - - getMockAdlFileSystem().removeAcl(new Path("/test1/test2")); - } - - @Test(expected=AccessControlException.class) - public void testSetAcl() throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - List entries = new ArrayList(); - AclEntry.Builder aclEntryBuilder = new AclEntry.Builder(); - aclEntryBuilder.setName("hadoop"); - aclEntryBuilder.setType(AclEntryType.USER); - aclEntryBuilder.setPermission(FsAction.ALL); - aclEntryBuilder.setScope(AclEntryScope.ACCESS); - entries.add(aclEntryBuilder.build()); - - aclEntryBuilder.setName("hdfs"); - aclEntryBuilder.setType(AclEntryType.GROUP); - aclEntryBuilder.setPermission(FsAction.READ_WRITE); - aclEntryBuilder.setScope(AclEntryScope.DEFAULT); - entries.add(aclEntryBuilder.build()); - - getMockAdlFileSystem().setAcl(new Path("/test1/test2"), entries); - - getMockServer().enqueue(new MockResponse().setResponseCode(403) - .setBody(TestADLResponseData.getAccessControlException())); - - getMockAdlFileSystem().setAcl(new Path("/test1/test2"), entries); - } - - @Test(expected=AccessControlException.class) - public void testCheckAccess() throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.ALL); - - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.EXECUTE); - - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.READ); - - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem() - .access(new Path("/test1/test2"), FsAction.READ_EXECUTE); - - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem() - .access(new Path("/test1/test2"), FsAction.READ_WRITE); - - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.NONE); - - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.WRITE); - - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem() - .access(new Path("/test1/test2"), FsAction.WRITE_EXECUTE); - - getMockServer().enqueue(new MockResponse().setResponseCode(403) - .setBody(TestADLResponseData.getAccessControlException())); - - getMockAdlFileSystem() - .access(new Path("/test1/test2"), FsAction.WRITE_EXECUTE); - } - - @Test(expected=AccessControlException.class) - public void testSetPermission() throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem() - .setPermission(new Path("/test1/test2"), FsPermission.getDefault()); - - getMockServer().enqueue(new MockResponse().setResponseCode(403) - .setBody(TestADLResponseData.getAccessControlException())); - - getMockAdlFileSystem() - .setPermission(new Path("/test1/test2"), FsPermission.getDefault()); - } - - @Test(expected=AccessControlException.class) - public void testSetOwner() throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200)); - getMockAdlFileSystem().setOwner(new Path("/test1/test2"), "hadoop", "hdfs"); - - getMockServer().enqueue(new MockResponse().setResponseCode(403) - .setBody(TestADLResponseData.getAccessControlException())); - - getMockAdlFileSystem() - .setOwner(new Path("/test1/test2"), "hadoop", "hdfs"); - } - - @Test - public void getAclStatusAsExpected() throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getGetAclStatusJSONResponse())); - AclStatus aclStatus = getMockAdlFileSystem() - .getAclStatus(new Path("/test1/test2")); - Assert.assertEquals(aclStatus.getGroup(), "supergroup"); - Assert.assertEquals(aclStatus.getOwner(), "hadoop"); - Assert.assertEquals((Short) aclStatus.getPermission().toShort(), - Short.valueOf("775", 8)); - - for (AclEntry entry : aclStatus.getEntries()) { - if (!(entry.toString().equalsIgnoreCase("user:carla:rw-") || entry - .toString().equalsIgnoreCase("group::r-x"))) { - Assert.fail("Unexpected entry : " + entry.toString()); - } - } - } - - @Test(expected=FileNotFoundException.class) - public void getAclStatusNotExists() throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(404) - .setBody(TestADLResponseData.getFileNotFoundException())); - - getMockAdlFileSystem().getAclStatus(new Path("/test1/test2")); - } - - @Test(expected=AccessControlException.class) - public void testAclStatusDenied() throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(403) - .setBody(TestADLResponseData.getAccessControlException())); - - getMockAdlFileSystem().getAclStatus(new Path("/test1/test2")); - } -} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAdlRead.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAdlRead.java deleted file mode 100644 index 172663cbd84..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAdlRead.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.adl; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.adl.common.Parallelized; -import org.apache.hadoop.fs.adl.common.TestDataForRead; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.ByteArrayInputStream; -import java.io.EOFException; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Random; - -import static org.apache.hadoop.fs.adl.AdlConfKeys.READ_AHEAD_BUFFER_SIZE_KEY; - -/** - * This class is responsible for stress positional reads vs number of network - * calls required by to fetch the amount of data. Test does ensure the data - * integrity and order of the data is maintained. - */ -@RunWith(Parallelized.class) -public class TestAdlRead extends AdlMockWebServer { - - private TestDataForRead testData; - - public TestAdlRead(TestDataForRead testData) { - Configuration configuration = new Configuration(); - configuration.setInt(READ_AHEAD_BUFFER_SIZE_KEY, 4 * 1024); - setConf(configuration); - this.testData = testData; - } - - @Parameterized.Parameters(name = "{index}") - public static Collection testDataForReadOperation() { - return Arrays.asList(new Object[][] { - - //-------------------------- - // Test Data - //-------------------------- - {new TestDataForRead("Hello World".getBytes(), 2, 1000, true)}, - {new TestDataForRead( - ("the problem you appear to be wrestling with is that this doesn't " - + "display very well. ").getBytes(), 2, 1000, true)}, - {new TestDataForRead(("您的數據是寶貴的資產，以您的組織，並有當前和未來價值。由於這個原因，" - + "所有的數據應存儲以供將來分析。今天，這往往是不這樣做，" + "因為傳統的分析基礎架構的限制，" - + "像模式的預定義，存儲大數據集和不同的數據筒倉的傳播的成本。" - + "為了應對這一挑戰，數據湖面概念被引入作為一個企業級存儲庫來存儲所有" - + "類型的在一個地方收集到的數據。對於運作和探索性分析的目的，所有類型的" + "數據可以定義需求或模式之前被存儲在數據湖。") - .getBytes(), 2, 1000, true)}, {new TestDataForRead( - TestADLResponseData.getRandomByteArrayData(4 * 1024), 2, 10, true)}, - {new TestDataForRead(TestADLResponseData.getRandomByteArrayData(100), 2, - 1000, true)}, {new TestDataForRead( - TestADLResponseData.getRandomByteArrayData(1 * 1024), 2, 50, true)}, - {new TestDataForRead( - TestADLResponseData.getRandomByteArrayData(8 * 1024), 3, 10, - false)}, {new TestDataForRead( - TestADLResponseData.getRandomByteArrayData(16 * 1024), 5, 10, false)}, - {new TestDataForRead( - TestADLResponseData.getRandomByteArrayData(32 * 1024), 9, 10, - false)}, {new TestDataForRead( - TestADLResponseData.getRandomByteArrayData(64 * 1024), 17, 10, - false)}}); - } - - @Test - public void testEntireBytes() throws IOException, InterruptedException { - getMockServer().setDispatcher(testData.getDispatcher()); - FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test")); - byte[] expectedData = new byte[testData.getActualData().length]; - int n = 0; - int len = expectedData.length; - int off = 0; - while (n < len) { - int count = in.read(expectedData, off + n, len - n); - if (count < 0) { - throw new EOFException(); - } - n += count; - } - - Assert.assertEquals(expectedData.length, testData.getActualData().length); - Assert.assertArrayEquals(expectedData, testData.getActualData()); - in.close(); - if (testData.isCheckOfNoOfCalls()) { - Assert.assertEquals(testData.getExpectedNoNetworkCall(), - getMockServer().getRequestCount()); - } - } - - @Test - public void testSeekOperation() throws IOException, InterruptedException { - getMockServer().setDispatcher(testData.getDispatcher()); - FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test")); - Random random = new Random(); - for (int i = 0; i < 1000; ++i) { - int position = random.nextInt(testData.getActualData().length); - in.seek(position); - Assert.assertEquals(position, in.getPos()); - Assert.assertEquals(testData.getActualData()[position] & 0xFF, in.read()); - } - in.close(); - if (testData.isCheckOfNoOfCalls()) { - Assert.assertEquals(testData.getExpectedNoNetworkCall(), - getMockServer().getRequestCount()); - } - } - - @Test - public void testReadServerCalls() throws IOException, InterruptedException { - getMockServer().setDispatcher(testData.getDispatcher()); - FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test")); - byte[] expectedData = new byte[testData.getActualData().length]; - in.readFully(expectedData); - Assert.assertArrayEquals(expectedData, testData.getActualData()); - Assert.assertEquals(testData.getExpectedNoNetworkCall(), - getMockServer().getRequestCount()); - in.close(); - } - - @Test - public void testReadFully() throws IOException, InterruptedException { - getMockServer().setDispatcher(testData.getDispatcher()); - FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test")); - byte[] expectedData = new byte[testData.getActualData().length]; - in.readFully(expectedData); - Assert.assertArrayEquals(expectedData, testData.getActualData()); - - in.readFully(0, expectedData); - Assert.assertArrayEquals(expectedData, testData.getActualData()); - - in.readFully(0, expectedData, 0, expectedData.length); - Assert.assertArrayEquals(expectedData, testData.getActualData()); - in.close(); - } - - @Test - public void testRandomPositionalReadUsingReadFully() - throws IOException, InterruptedException { - getMockServer().setDispatcher(testData.getDispatcher()); - FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test")); - ByteArrayInputStream actualData = new ByteArrayInputStream( - testData.getActualData()); - Random random = new Random(); - for (int i = 0; i < testData.getIntensityOfTest(); ++i) { - int offset = random.nextInt(testData.getActualData().length); - int length = testData.getActualData().length - offset; - byte[] expectedData = new byte[length]; - byte[] actualDataSubset = new byte[length]; - actualData.reset(); - actualData.skip(offset); - actualData.read(actualDataSubset, 0, length); - - in.readFully(offset, expectedData, 0, length); - Assert.assertArrayEquals(expectedData, actualDataSubset); - } - - for (int i = 0; i < testData.getIntensityOfTest(); ++i) { - int offset = random.nextInt(testData.getActualData().length); - int length = random.nextInt(testData.getActualData().length - offset); - byte[] expectedData = new byte[length]; - byte[] actualDataSubset = new byte[length]; - actualData.reset(); - actualData.skip(offset); - actualData.read(actualDataSubset, 0, length); - - in.readFully(offset, expectedData, 0, length); - Assert.assertArrayEquals(expectedData, actualDataSubset); - } - - in.close(); - } -} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestConcurrentDataReadOperations.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestConcurrentDataReadOperations.java deleted file mode 100644 index b7905627831..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestConcurrentDataReadOperations.java +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.adl; - -import com.squareup.okhttp.mockwebserver.Dispatcher; -import com.squareup.okhttp.mockwebserver.MockResponse; -import com.squareup.okhttp.mockwebserver.RecordedRequest; -import okio.Buffer; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.Path; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * This class is responsible for testing multiple threads trying to access same - * or multiple files from the offset. - */ -@RunWith(Parameterized.class) -public class TestConcurrentDataReadOperations extends AdlMockWebServer { - private static final Logger LOG = LoggerFactory - .getLogger(TestConcurrentDataReadOperations.class); - private static final Object LOCK = new Object(); - private static FSDataInputStream commonHandle = null; - private int concurrencyLevel; - - public TestConcurrentDataReadOperations(int concurrencyLevel) { - this.concurrencyLevel = concurrencyLevel; - } - - @Parameterized.Parameters(name = "{index}") - public static Collection testDataNumberOfConcurrentRun() { - return Arrays.asList(new Object[][] {{1}, {2}, {3}, {4}, {5}}); - } - - public static byte[] getRandomByteArrayData(int size) { - byte[] b = new byte[size]; - Random rand = new Random(); - rand.nextBytes(b); - return b; - } - - private void setDispatcher(final ArrayList testData) { - getMockServer().setDispatcher(new Dispatcher() { - @Override - public MockResponse dispatch(RecordedRequest recordedRequest) - throws InterruptedException { - CreateTestData currentRequest = null; - for (CreateTestData local : testData) { - if (recordedRequest.getPath().contains(local.path.toString())) { - currentRequest = local; - break; - } - } - - if (currentRequest == null) { - new MockResponse().setBody("Request data not found") - .setResponseCode(501); - } - - if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) { - return new MockResponse().setResponseCode(200).setBody( - TestADLResponseData - .getGetFileStatusJSONResponse(currentRequest.data.length)); - } - - if (recordedRequest.getRequestLine().contains("op=OPEN")) { - String request = recordedRequest.getRequestLine(); - int offset = 0; - int byteCount = 0; - - Pattern pattern = Pattern.compile("offset=([0-9]+)"); - Matcher matcher = pattern.matcher(request); - if (matcher.find()) { - LOG.debug(matcher.group(1)); - offset = Integer.parseInt(matcher.group(1)); - } - - pattern = Pattern.compile("length=([0-9]+)"); - matcher = pattern.matcher(request); - if (matcher.find()) { - LOG.debug(matcher.group(1)); - byteCount = Integer.parseInt(matcher.group(1)); - } - - Buffer buf = new Buffer(); - buf.write(currentRequest.data, offset, - Math.min(currentRequest.data.length - offset, byteCount)); - return new MockResponse().setResponseCode(200) - .setChunkedBody(buf, 4 * 1024 * 1024); - } - - return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501); - } - }); - } - - @Before - public void resetHandle() { - commonHandle = null; - } - - @Test - public void testParallelReadOnDifferentStreams() - throws IOException, InterruptedException, ExecutionException { - - ArrayList createTestData = new ArrayList(); - - Random random = new Random(); - - for (int i = 0; i < concurrencyLevel; i++) { - CreateTestData testData = new CreateTestData(); - testData - .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()), - getRandomByteArrayData(random.nextInt(1 * 1024 * 1024))); - createTestData.add(testData); - } - - setDispatcher(createTestData); - - ArrayList readTestData = new ArrayList(); - for (CreateTestData local : createTestData) { - ReadTestData localReadData = new ReadTestData(); - localReadData.set(local.path, local.data, 0); - readTestData.add(localReadData); - } - - runReadTest(readTestData, false); - } - - @Test - public void testParallelReadOnSameStreams() - throws IOException, InterruptedException, ExecutionException { - ArrayList createTestData = new ArrayList(); - - Random random = new Random(); - - for (int i = 0; i < 1; i++) { - CreateTestData testData = new CreateTestData(); - testData - .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()), - getRandomByteArrayData(1024 * 1024)); - createTestData.add(testData); - } - - setDispatcher(createTestData); - - ArrayList readTestData = new ArrayList(); - ByteArrayInputStream buffered = new ByteArrayInputStream( - createTestData.get(0).data); - - ReadTestData readInitially = new ReadTestData(); - byte[] initialData = new byte[1024 * 1024]; - buffered.read(initialData); - - readInitially.set(createTestData.get(0).path, initialData, 0); - readTestData.add(readInitially); - runReadTest(readTestData, false); - - readTestData.clear(); - - for (int i = 0; i < concurrencyLevel * 5; i++) { - ReadTestData localReadData = new ReadTestData(); - int offset = random.nextInt((1024 * 1024) - 1); - int length = 1024 * 1024 - offset; - byte[] expectedData = new byte[length]; - buffered.reset(); - buffered.skip(offset); - buffered.read(expectedData); - localReadData.set(createTestData.get(0).path, expectedData, offset); - readTestData.add(localReadData); - } - - runReadTest(readTestData, true); - } - - void runReadTest(ArrayList testData, boolean useSameStream) - throws InterruptedException, ExecutionException { - - ExecutorService executor = Executors.newFixedThreadPool(testData.size()); - Future[] subtasks = new Future[testData.size()]; - - for (int i = 0; i < testData.size(); i++) { - subtasks[i] = executor.submit( - new ReadConcurrentRunnable(testData.get(i).data, testData.get(i).path, - testData.get(i).offset, useSameStream)); - } - - executor.shutdown(); - - // wait until all tasks are finished - executor.awaitTermination(120, TimeUnit.SECONDS); - - for (int i = 0; i < testData.size(); ++i) { - Assert.assertTrue((Boolean) subtasks[i].get()); - } - } - - class ReadTestData { - private Path path; - private byte[] data; - private int offset; - - public void set(Path filePath, byte[] dataToBeRead, int fromOffset) { - this.path = filePath; - this.data = dataToBeRead; - this.offset = fromOffset; - } - } - - class CreateTestData { - private Path path; - private byte[] data; - - public void set(Path filePath, byte[] dataToBeWritten) { - this.path = filePath; - this.data = dataToBeWritten; - } - } - - class ReadConcurrentRunnable implements Callable { - private Path path; - private int offset; - private byte[] expectedData; - private boolean useSameStream; - - public ReadConcurrentRunnable(byte[] expectedData, Path path, int offset, - boolean useSameStream) { - this.path = path; - this.offset = offset; - this.expectedData = expectedData; - this.useSameStream = useSameStream; - } - - public Boolean call() throws IOException { - try { - FSDataInputStream in; - if (useSameStream) { - synchronized (LOCK) { - if (commonHandle == null) { - commonHandle = getMockAdlFileSystem().open(path); - } - in = commonHandle; - } - } else { - in = getMockAdlFileSystem().open(path); - } - - byte[] actualData = new byte[expectedData.length]; - in.readFully(offset, actualData); - Assert.assertArrayEquals("Path :" + path.toString() + " did not match.", - expectedData, actualData); - if (!useSameStream) { - in.close(); - } - } catch (IOException e) { - e.printStackTrace(); - return false; - } - return true; - } - } -} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java deleted file mode 100644 index 737534cf010..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.adl; - -import com.squareup.okhttp.mockwebserver.MockResponse; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider; -import org.apache.hadoop.fs.permission.FsPermission; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.Collection; - -import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE; -import static org.apache.hadoop.fs.adl.AdlConfKeys - .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; -import static org.apache.hadoop.fs.adl.AdlConfKeys - .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; - -/** - * Test access token provider behaviour with custom token provider and for token - * provider cache is enabled. - */ -@RunWith(Parameterized.class) -public class TestCustomTokenProvider extends AdlMockWebServer { - private static final long TEN_MINUTES_IN_MILIS = 600000; - private int backendCallCount; - private int expectedCallbackToAccessToken; - private TestableAdlFileSystem[] fileSystems; - private Class typeOfTokenProviderClass; - private long expiryFromNow; - private int fsObjectCount; - - public TestCustomTokenProvider(Class typeOfTokenProviderClass, - long expiryFromNow, int fsObjectCount, int backendCallCount, - int expectedCallbackToAccessToken) - throws IllegalAccessException, InstantiationException, URISyntaxException, - IOException { - this.typeOfTokenProviderClass = typeOfTokenProviderClass; - this.expiryFromNow = expiryFromNow; - this.fsObjectCount = fsObjectCount; - this.backendCallCount = backendCallCount; - this.expectedCallbackToAccessToken = expectedCallbackToAccessToken; - } - - @Parameterized.Parameters(name = "{index}") - public static Collection testDataForTokenProvider() { - return Arrays.asList(new Object[][] { - // Data set in order - // INPUT - CustomTokenProvider class to load - // INPUT - expiry time in milis. Subtract from current time - // INPUT - No. of FileSystem object - // INPUT - No. of backend calls per FileSystem object - // EXPECTED - Number of callbacks to get token after test finished. - {CustomMockTokenProvider.class, 0, 1, 1, 1}, - {CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 1, 1, 1}, - {CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 2, 1, 2}, - {CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 10, 10, 10}}); - } - - /** - * Explicitly invoked init so that base class mock server is setup before - * test data initialization is done. - * - * @throws IOException - * @throws URISyntaxException - */ - public void init() throws IOException, URISyntaxException { - Configuration configuration = new Configuration(); - configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, - TokenProviderType.Custom); - configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, - typeOfTokenProviderClass.getName()); - fileSystems = new TestableAdlFileSystem[fsObjectCount]; - URI uri = new URI("adl://localhost:" + getPort()); - - for (int i = 0; i < fsObjectCount; ++i) { - fileSystems[i] = new TestableAdlFileSystem(); - fileSystems[i].initialize(uri, configuration); - - ((CustomMockTokenProvider) fileSystems[i].getAzureTokenProvider()) - .setExpiryTimeInMillisAfter(expiryFromNow); - } - } - - @Test - public void testCustomTokenManagement() - throws IOException, URISyntaxException { - int accessTokenCallbackDuringExec = 0; - init(); - for (TestableAdlFileSystem tfs : fileSystems) { - for (int i = 0; i < backendCallCount; ++i) { - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getGetFileStatusJSONResponse())); - FileStatus fileStatus = tfs.getFileStatus(new Path("/test1/test2")); - Assert.assertTrue(fileStatus.isFile()); - Assert.assertEquals("adl://" + getMockServer().getHostName() + ":" + - getMockServer().getPort() + "/test1/test2", - fileStatus.getPath().toString()); - Assert.assertEquals(4194304, fileStatus.getLen()); - Assert.assertEquals(ADL_BLOCK_SIZE, fileStatus.getBlockSize()); - Assert.assertEquals(1, fileStatus.getReplication()); - Assert - .assertEquals(new FsPermission("777"), fileStatus.getPermission()); - Assert.assertEquals("NotSupportYet", fileStatus.getOwner()); - Assert.assertEquals("NotSupportYet", fileStatus.getGroup()); - } - - accessTokenCallbackDuringExec += ((CustomMockTokenProvider) tfs - .getAzureTokenProvider()).getAccessTokenRequestCount(); - } - - Assert.assertEquals(expectedCallbackToAccessToken, - accessTokenCallbackDuringExec); - } -} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java deleted file mode 100644 index 95c23639564..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.fs.adl; - -import com.squareup.okhttp.mockwebserver.MockResponse; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.util.Time; -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URISyntaxException; - -import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE; - -/** - * This class is responsible for testing local getFileStatus implementation - * to cover correct parsing of successful and error JSON response - * from the server. - * Adls GetFileStatus operation is in detail covered in - * org.apache.hadoop.fs.adl.live testing package. - */ -public class TestGetFileStatus extends AdlMockWebServer { - private static final Logger LOG = - LoggerFactory.getLogger(TestGetFileStatus.class); - - @Test - public void getFileStatusReturnsAsExpected() - throws URISyntaxException, IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getGetFileStatusJSONResponse())); - long startTime = Time.monotonicNow(); - Path path = new Path("/test1/test2"); - FileStatus fileStatus = getMockAdlFileSystem().getFileStatus(path); - long endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - Assert.assertTrue(fileStatus.isFile()); - Assert.assertEquals("adl://" + getMockServer().getHostName() + ":" + - getMockServer().getPort() + "/test1/test2", - fileStatus.getPath().toString()); - Assert.assertEquals(4194304, fileStatus.getLen()); - Assert.assertEquals(ADL_BLOCK_SIZE, fileStatus.getBlockSize()); - Assert.assertEquals(1, fileStatus.getReplication()); - Assert.assertEquals(new FsPermission("777"), fileStatus.getPermission()); - Assert.assertEquals("NotSupportYet", fileStatus.getOwner()); - Assert.assertEquals("NotSupportYet", fileStatus.getGroup()); - Assert.assertTrue(path + " should have Acl!", fileStatus.hasAcl()); - Assert.assertFalse(path + " should not be encrypted!", - fileStatus.isEncrypted()); - Assert.assertFalse(path + " should not be erasure coded!", - fileStatus.isErasureCoded()); - } - - @Test - public void getFileStatusAclBit() throws URISyntaxException, IOException { - // With ACLBIT set to true - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getGetFileStatusJSONResponse(true))); - long startTime = Time.monotonicNow(); - FileStatus fileStatus = getMockAdlFileSystem() - .getFileStatus(new Path("/test1/test2")); - long endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - Assert.assertTrue(fileStatus.isFile()); - Assert.assertTrue(fileStatus.hasAcl()); - Assert.assertTrue(fileStatus.getPermission().getAclBit()); - - // With ACLBIT set to false - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getGetFileStatusJSONResponse(false))); - startTime = Time.monotonicNow(); - fileStatus = getMockAdlFileSystem() - .getFileStatus(new Path("/test1/test2")); - endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - Assert.assertTrue(fileStatus.isFile()); - Assert.assertFalse(fileStatus.hasAcl()); - Assert.assertFalse(fileStatus.getPermission().getAclBit()); - } - -} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java deleted file mode 100644 index db32476f459..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.fs.adl; - -import com.squareup.okhttp.mockwebserver.MockResponse; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.Time; -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URISyntaxException; - -/** - * This class is responsible for testing local listStatus implementation to - * cover correct parsing of successful and error JSON response from the server. - * Adls ListStatus functionality is in detail covered in - * org.apache.hadoop.fs.adl.live testing package. - */ -public class TestListStatus extends AdlMockWebServer { - - private static final Logger LOG = LoggerFactory - .getLogger(TestListStatus.class); - - @Test - public void listStatusReturnsAsExpected() throws IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getListFileStatusJSONResponse(10))); - long startTime = Time.monotonicNow(); - FileStatus[] ls = getMockAdlFileSystem() - .listStatus(new Path("/test1/test2")); - long endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - Assert.assertEquals(10, ls.length); - - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getListFileStatusJSONResponse(200))); - startTime = Time.monotonicNow(); - ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2")); - endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - Assert.assertEquals(200, ls.length); - - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getListFileStatusJSONResponse(2048))); - startTime = Time.monotonicNow(); - ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2")); - endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - Assert.assertEquals(2048, ls.length); - } - - @Test - public void listStatusOnFailure() throws IOException { - getMockServer().enqueue(new MockResponse().setResponseCode(403).setBody( - TestADLResponseData.getErrorIllegalArgumentExceptionJSONResponse())); - FileStatus[] ls = null; - long startTime = Time.monotonicNow(); - try { - ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2")); - } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("Invalid")); - } - long endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - - // SDK may increase number of retry attempts before error is propagated - // to caller. Adding max 10 error responses in the queue to align with SDK. - for (int i = 0; i < 10; ++i) { - getMockServer().enqueue(new MockResponse().setResponseCode(500).setBody( - TestADLResponseData.getErrorInternalServerExceptionJSONResponse())); - } - - startTime = Time.monotonicNow(); - try { - ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2")); - } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("Internal Server Error")); - } - endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - } - - @Test - public void listStatusAcl() - throws URISyntaxException, IOException { - // With ACLBIT set to true - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getListFileStatusJSONResponse(true))); - FileStatus[] ls = null; - long startTime = Time.monotonicNow(); - ls = getMockAdlFileSystem() - .listStatus(new Path("/test1/test2")); - long endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - for (int i = 0; i < ls.length; i++) { - Assert.assertTrue(ls[i].isDirectory()); - Assert.assertTrue(ls[i].hasAcl()); - Assert.assertTrue(ls[i].getPermission().getAclBit()); - } - - // With ACLBIT set to false - ls = null; - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getListFileStatusJSONResponse(false))); - startTime = Time.monotonicNow(); - ls = getMockAdlFileSystem() - .listStatus(new Path("/test1/test2")); - endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - for (int i = 0; i < ls.length; i++) { - Assert.assertTrue(ls[i].isDirectory()); - Assert.assertFalse(ls[i].hasAcl()); - Assert.assertFalse(ls[i].getPermission().getAclBit()); - } - } -} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java deleted file mode 100644 index 4acb39b196b..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.fs.adl; - -/** - * Mock adl file storage subclass to mock adl storage on local http service. - */ -public class TestableAdlFileSystem extends AdlFileSystem { - @Override - protected String getTransportScheme() { - return "http"; - } -} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/ExpectedResponse.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/ExpectedResponse.java deleted file mode 100644 index dc8577db697..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/ExpectedResponse.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.fs.adl.common; - -import com.squareup.okhttp.mockwebserver.MockResponse; - -import java.util.ArrayList; - -/** - * Supporting class to hold expected MockResponse object along with parameters - * for validation in test methods. - */ -public class ExpectedResponse { - private MockResponse response; - private ArrayList expectedQueryParameters = new ArrayList(); - private int expectedBodySize; - private String httpRequestType; - - public int getExpectedBodySize() { - return expectedBodySize; - } - - public String getHttpRequestType() { - return httpRequestType; - } - - public ArrayList getExpectedQueryParameters() { - return expectedQueryParameters; - } - - public MockResponse getResponse() { - return response; - } - - ExpectedResponse set(MockResponse mockResponse) { - this.response = mockResponse; - return this; - } - - ExpectedResponse addExpectedQueryParam(String param) { - expectedQueryParameters.add(param); - return this; - } - - ExpectedResponse addExpectedBodySize(int bodySize) { - this.expectedBodySize = bodySize; - return this; - } - - ExpectedResponse addExpectedHttpRequestType(String expectedHttpRequestType) { - this.httpRequestType = expectedHttpRequestType; - return this; - } -} diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/TestDataForRead.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/TestDataForRead.java deleted file mode 100644 index 509b3f041b3..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/TestDataForRead.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.fs.adl.common; - -import com.squareup.okhttp.mockwebserver.Dispatcher; -import com.squareup.okhttp.mockwebserver.MockResponse; -import com.squareup.okhttp.mockwebserver.RecordedRequest; -import okio.Buffer; -import org.apache.hadoop.fs.adl.TestADLResponseData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Supporting class for mock test to validate Adls read operation. - */ -public class TestDataForRead { - private static final Logger LOG = LoggerFactory - .getLogger(TestDataForRead.class); - - private byte[] actualData; - private ArrayList responses; - private Dispatcher dispatcher; - private int intensityOfTest; - private boolean checkOfNoOfCalls; - private int expectedNoNetworkCall; - - public TestDataForRead(final byte[] actualData, int expectedNoNetworkCall, - int intensityOfTest, boolean checkOfNoOfCalls) { - - this.checkOfNoOfCalls = checkOfNoOfCalls; - this.actualData = actualData; - responses = new ArrayList(); - this.expectedNoNetworkCall = expectedNoNetworkCall; - this.intensityOfTest = intensityOfTest; - - dispatcher = new Dispatcher() { - @Override - public MockResponse dispatch(RecordedRequest recordedRequest) - throws InterruptedException { - - if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) { - return new MockResponse().setResponseCode(200).setBody( - TestADLResponseData - .getGetFileStatusJSONResponse(actualData.length)); - } - - if (recordedRequest.getRequestLine().contains("op=OPEN")) { - String request = recordedRequest.getRequestLine(); - int offset = 0; - int byteCount = 0; - - Pattern pattern = Pattern.compile("offset=([0-9]+)"); - Matcher matcher = pattern.matcher(request); - if (matcher.find()) { - LOG.debug(matcher.group(1)); - offset = Integer.parseInt(matcher.group(1)); - } - - pattern = Pattern.compile("length=([0-9]+)"); - matcher = pattern.matcher(request); - if (matcher.find()) { - LOG.debug(matcher.group(1)); - byteCount = Integer.parseInt(matcher.group(1)); - } - - Buffer buf = new Buffer(); - buf.write(actualData, offset, - Math.min(actualData.length - offset, byteCount)); - return new MockResponse().setResponseCode(200) - .setChunkedBody(buf, 4 * 1024 * 1024); - } - - return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501); - } - }; - } - - public boolean isCheckOfNoOfCalls() { - return checkOfNoOfCalls; - } - - public int getExpectedNoNetworkCall() { - return expectedNoNetworkCall; - } - - public int getIntensityOfTest() { - return intensityOfTest; - } - - public byte[] getActualData() { - return actualData; - } - - public ArrayList getResponses() { - return responses; - } - - public Dispatcher getDispatcher() { - return dispatcher; - } -}