diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index f3167f283aa..c51030bb2c1 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1772,19 +1772,6 @@ - - - fs.o3.impl - org.apache.hadoop.fs.ozone.OzoneFileSystem - The implementation class of the Ozone FileSystem. - - - - fs.AbstractFileSystem.o3.impl - org.apache.hadoop.fs.ozone.OzFs - The implementation class of the OzFs AbstractFileSystem. - - diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index e10617daaba..50af23004be 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -102,7 +102,7 @@ public void initializeMemberVariables() { xmlPrefixToSkipCompare.add("fs.s3a."); // O3 properties are in a different subtree. - xmlPrefixToSkipCompare.add("fs.o3."); + xmlPrefixToSkipCompare.add("fs.o3fs."); //ftp properties are in a different subtree. // - org.apache.hadoop.fs.ftp.FTPFileSystem. diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 8ccc6480a86..b77d6211a91 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -63,7 +63,10 @@ public final class OzoneConsts { public static final String OZONE_USER = "user"; public static final String OZONE_REQUEST = "request"; - public static final String OZONE_URI_SCHEME = "o3"; + // Ozone File System scheme + public static final String OZONE_URI_SCHEME = "o3fs"; + + public static final String OZONE_RPC_SCHEME = "o3"; public static final String OZONE_HTTP_SCHEME = "http"; public static final String OZONE_URI_DELIMITER = "/"; diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config index 675dcba0538..5061afaf832 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem +CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem OZONE-SITE.XML_ozone.om.address=ozoneManager OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 OZONE-SITE.XML_ozone.scm.names=scm diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot index 3720d011a3b..fd34e359387 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot @@ -30,83 +30,83 @@ Create volume and bucket Execute ozone sh bucket create http://ozoneManager/fstest2/bucket3 Check volume from ozonefs - ${result} = Execute ozone fs -ls o3://bucket1.fstest/ + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/ Run ozoneFS tests - Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} testdir/deep - Execute ozone fs -copyFromLocal NOTICE.txt o3://bucket1.fstest/testdir/deep/ + Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/ ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} NOTICE.txt - Execute ozone fs -put NOTICE.txt o3://bucket1.fstest/testdir/deep/PUTFILE.txt + Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} PUTFILE.txt - ${result} = Execute ozone fs -ls o3://bucket1.fstest/testdir/deep/ + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/ Should contain ${result} NOTICE.txt Should contain ${result} PUTFILE.txt - Execute ozone fs -mv o3://bucket1.fstest/testdir/deep/NOTICE.txt o3://bucket1.fstest/testdir/deep/MOVED.TXT + Execute ozone fs -mv o3fs://bucket1.fstest/testdir/deep/NOTICE.txt o3fs://bucket1.fstest/testdir/deep/MOVED.TXT ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} MOVED.TXT Should not contain ${result} NOTICE.txt - Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep/subdir1 - Execute ozone fs -cp o3://bucket1.fstest/testdir/deep/MOVED.TXT o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep/subdir1 + Execute ozone fs -cp o3fs://bucket1.fstest/testdir/deep/MOVED.TXT o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} subdir1/NOTICE.txt - ${result} = Execute ozone fs -ls o3://bucket1.fstest/testdir/deep/subdir1/ + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/subdir1/ Should contain ${result} NOTICE.txt - Execute ozone fs -cat o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + Execute ozone fs -cat o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt Should not contain ${result} Failed - Execute ozone fs -rm o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + Execute ozone fs -rm o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should not contain ${result} NOTICE.txt - ${result} = Execute ozone fs -rmdir o3://bucket1.fstest/testdir/deep/subdir1/ + ${result} = Execute ozone fs -rmdir o3fs://bucket1.fstest/testdir/deep/subdir1/ ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should not contain ${result} subdir1 - Execute ozone fs -touch o3://bucket1.fstest/testdir/TOUCHFILE.txt + Execute ozone fs -touch o3fs://bucket1.fstest/testdir/TOUCHFILE.txt ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} TOUCHFILE.txt - Execute ozone fs -rm -r o3://bucket1.fstest/testdir/ + Execute ozone fs -rm -r o3fs://bucket1.fstest/testdir/ ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should not contain ${result} testdir Execute rm -Rf localdir1 Execute mkdir localdir1 Execute cp NOTICE.txt localdir1/LOCAL.txt - Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir1 - Execute ozone fs -copyFromLocal localdir1 o3://bucket1.fstest/testdir1/ - Execute ozone fs -put NOTICE.txt o3://bucket1.fstest/testdir1/NOTICE.txt + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir1 + Execute ozone fs -copyFromLocal localdir1 o3fs://bucket1.fstest/testdir1/ + Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir1/NOTICE.txt - ${result} = Execute ozone fs -ls -R o3://bucket1.fstest/testdir1/ + ${result} = Execute ozone fs -ls -R o3fs://bucket1.fstest/testdir1/ Should contain ${result} localdir1/LOCAL.txt Should contain ${result} testdir1/NOTICE.txt - Execute ozone fs -mkdir -p o3://bucket2.fstest/testdir2 - Execute ozone fs -mkdir -p o3://bucket3.fstest2/testdir3 + Execute ozone fs -mkdir -p o3fs://bucket2.fstest/testdir2 + Execute ozone fs -mkdir -p o3fs://bucket3.fstest2/testdir3 - Execute ozone fs -cp o3://bucket1.fstest/testdir1/localdir1 o3://bucket2.fstest/testdir2/ + Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket2.fstest/testdir2/ - Execute ozone fs -cp o3://bucket1.fstest/testdir1/localdir1 o3://bucket3.fstest2/testdir3/ + Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket3.fstest2/testdir3/ Execute ozone sh key put o3://ozoneManager/fstest/bucket1/KEY.txt NOTICE.txt - ${result} = Execute ozone fs -ls o3://bucket1.fstest/KEY.txt + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/KEY.txt Should contain ${result} KEY.txt - ${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3://bucket1.fstest/KEY.txt + ${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/KEY.txt Should Be Equal As Integers ${rc} 1 Should contain ${result} File exists Execute rm -Rf GET.txt - Execute ozone fs -get o3://bucket1.fstest/KEY.txt GET.txt + Execute ozone fs -get o3fs://bucket1.fstest/KEY.txt GET.txt Execute ls -l GET.txt - ${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3://abcde.pqrs/ + ${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3fs://abcde.pqrs/ Should Be Equal As Integers ${rc} 1 Should contain ${result} VOLUME_NOT_FOUND diff --git a/hadoop-ozone/docs/content/OzoneFS.md b/hadoop-ozone/docs/content/OzoneFS.md index d0621bee6dd..e19c9b9ebd9 100644 --- a/hadoop-ozone/docs/content/OzoneFS.md +++ b/hadoop-ozone/docs/content/OzoneFS.md @@ -42,16 +42,16 @@ Please add the following entry to the core-site.xml. {{< highlight xml >}} - fs.o3.impl + fs.o3fs.impl org.apache.hadoop.fs.ozone.OzoneFileSystem fs.default.name - o3://localhost:9864/volume/bucket + o3fs://localhost:9864/volume/bucket {{< /highlight >}} -This will make this bucket to be the default file system for HDFS dfs commands and register the o3 file system type.. +This will make this bucket to be the default file system for HDFS dfs commands and register the o3fs file system type.. You also need to add the ozone-filesystem.jar file to the classpath: diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java index a9550c2e26f..5067855392b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java @@ -33,7 +33,7 @@ import org.apache.hadoop.ozone.client.rest.OzoneException; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_HTTP_SCHEME; -import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RPC_SCHEME; import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,7 +81,7 @@ protected URI verifyURI(String uri) Configuration conf = new OzoneConfiguration(); String scheme = ozoneURI.getScheme(); if (ozoneURI.getScheme() == null || scheme.isEmpty()) { - scheme = OZONE_URI_SCHEME; + scheme = OZONE_RPC_SCHEME; } if (scheme.equals(OZONE_HTTP_SCHEME)) { if (ozoneURI.getHost() != null) { @@ -94,7 +94,7 @@ protected URI verifyURI(String uri) } else { client = OzoneClientFactory.getRestClient(conf); } - } else if (scheme.equals(OZONE_URI_SCHEME)) { + } else if (scheme.equals(OZONE_RPC_SCHEME)) { if (ozoneURI.getHost() != null) { if (ozoneURI.getPort() == -1) { client = OzoneClientFactory.getRpcClient(ozoneURI.getHost()); diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java index 0b721fb0fbc..d52f3b707b8 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java @@ -107,7 +107,7 @@ public void initialize(URI name, Configuration conf) throws IOException { if (!matcher.matches()) { throw new IllegalArgumentException("Ozone file system url should be " - + "in the form o3://bucket.volume"); + + "in the form o3fs://bucket.volume"); } String bucketStr = matcher.group(1); String volumeStr = matcher.group(2); diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java index 217603102b5..0eb8dcedde8 100644 --- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java +++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java @@ -231,7 +231,7 @@ public void testPathToKey() throws Exception { ozoneFs.pathToKey(new Path("key1/key2"))); assertEquals("key1/key2", - ozoneFs.pathToKey(new Path("o3://test1/key1/key2"))); + ozoneFs.pathToKey(new Path("o3fs://test1/key1/key2"))); } private String getCurrentUser() {