Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-651. Rename o3 to o3fs for Filesystem.

This commit is contained in:
Jitendra Pandey 2018-10-17 14:18:46 -07:00
parent 9abda83947
commit d93d515af5
9 changed files with 40 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -1788,19 +1788,6 @@
</property> </property>
<!-- Ozone file system properties -->
<property>
<name>fs.o3.impl</name>
<value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
<description>The implementation class of the Ozone FileSystem.</description>
</property>
<property>
<name>fs.AbstractFileSystem.o3.impl</name>
<value>org.apache.hadoop.fs.ozone.OzFs</value>
<description>The implementation class of the OzFs AbstractFileSystem.</description>
</property>
<!-- ipc properties --> <!-- ipc properties -->
<property> <property>

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
View File

@ -103,7 +103,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPrefixToSkipCompare.add("fs.s3a."); xmlPrefixToSkipCompare.add("fs.s3a.");
// O3 properties are in a different subtree. // O3 properties are in a different subtree.
xmlPrefixToSkipCompare.add("fs.o3."); xmlPrefixToSkipCompare.add("fs.o3fs.");
//ftp properties are in a different subtree. //ftp properties are in a different subtree.
// - org.apache.hadoop.fs.ftp.FTPFileSystem. // - org.apache.hadoop.fs.ftp.FTPFileSystem.

5
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
View File

@ -63,7 +63,10 @@ public final class OzoneConsts {
public static final String OZONE_USER = "user"; public static final String OZONE_USER = "user";
public static final String OZONE_REQUEST = "request"; public static final String OZONE_REQUEST = "request";
public static final String OZONE_URI_SCHEME = "o3"; // Ozone File System scheme
public static final String OZONE_URI_SCHEME = "o3fs";
public static final String OZONE_RPC_SCHEME = "o3";
public static final String OZONE_HTTP_SCHEME = "http"; public static final String OZONE_HTTP_SCHEME = "http";
public static final String OZONE_URI_DELIMITER = "/"; public static final String OZONE_URI_DELIMITER = "/";

2
hadoop-ozone/dist/src/main/compose/ozonefs/docker-config vendored
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
OZONE-SITE.XML_ozone.om.address=ozoneManager OZONE-SITE.XML_ozone.om.address=ozoneManager
OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.names=scm

52
hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot vendored
View File

@ -30,83 +30,83 @@ Create volume and bucket
Execute ozone sh bucket create http://ozoneManager/fstest2/bucket3 Execute ozone sh bucket create http://ozoneManager/fstest2/bucket3
Check volume from ozonefs Check volume from ozonefs
${result} = Execute ozone fs -ls o3://bucket1.fstest/ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/
Run ozoneFS tests Run ozoneFS tests
Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} testdir/deep Should contain ${result} testdir/deep
Execute ozone fs -copyFromLocal NOTICE.txt o3://bucket1.fstest/testdir/deep/ Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} NOTICE.txt Should contain ${result} NOTICE.txt
Execute ozone fs -put NOTICE.txt o3://bucket1.fstest/testdir/deep/PUTFILE.txt Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} PUTFILE.txt Should contain ${result} PUTFILE.txt
${result} = Execute ozone fs -ls o3://bucket1.fstest/testdir/deep/ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/
Should contain ${result} NOTICE.txt Should contain ${result} NOTICE.txt
Should contain ${result} PUTFILE.txt Should contain ${result} PUTFILE.txt
Execute ozone fs -mv o3://bucket1.fstest/testdir/deep/NOTICE.txt o3://bucket1.fstest/testdir/deep/MOVED.TXT Execute ozone fs -mv o3fs://bucket1.fstest/testdir/deep/NOTICE.txt o3fs://bucket1.fstest/testdir/deep/MOVED.TXT
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} MOVED.TXT Should contain ${result} MOVED.TXT
Should not contain ${result} NOTICE.txt Should not contain ${result} NOTICE.txt
Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep/subdir1 Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep/subdir1
Execute ozone fs -cp o3://bucket1.fstest/testdir/deep/MOVED.TXT o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt Execute ozone fs -cp o3fs://bucket1.fstest/testdir/deep/MOVED.TXT o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} subdir1/NOTICE.txt Should contain ${result} subdir1/NOTICE.txt
${result} = Execute ozone fs -ls o3://bucket1.fstest/testdir/deep/subdir1/ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/subdir1/
Should contain ${result} NOTICE.txt Should contain ${result} NOTICE.txt
Execute ozone fs -cat o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt Execute ozone fs -cat o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
Should not contain ${result} Failed Should not contain ${result} Failed
Execute ozone fs -rm o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt Execute ozone fs -rm o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should not contain ${result} NOTICE.txt Should not contain ${result} NOTICE.txt
${result} = Execute ozone fs -rmdir o3://bucket1.fstest/testdir/deep/subdir1/ ${result} = Execute ozone fs -rmdir o3fs://bucket1.fstest/testdir/deep/subdir1/
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should not contain ${result} subdir1 Should not contain ${result} subdir1
Execute ozone fs -touch o3://bucket1.fstest/testdir/TOUCHFILE.txt Execute ozone fs -touch o3fs://bucket1.fstest/testdir/TOUCHFILE.txt
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} TOUCHFILE.txt Should contain ${result} TOUCHFILE.txt
Execute ozone fs -rm -r o3://bucket1.fstest/testdir/ Execute ozone fs -rm -r o3fs://bucket1.fstest/testdir/
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should not contain ${result} testdir Should not contain ${result} testdir
Execute rm -Rf localdir1 Execute rm -Rf localdir1
Execute mkdir localdir1 Execute mkdir localdir1
Execute cp NOTICE.txt localdir1/LOCAL.txt Execute cp NOTICE.txt localdir1/LOCAL.txt
Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir1 Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir1
Execute ozone fs -copyFromLocal localdir1 o3://bucket1.fstest/testdir1/ Execute ozone fs -copyFromLocal localdir1 o3fs://bucket1.fstest/testdir1/
Execute ozone fs -put NOTICE.txt o3://bucket1.fstest/testdir1/NOTICE.txt Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir1/NOTICE.txt
${result} = Execute ozone fs -ls -R o3://bucket1.fstest/testdir1/ ${result} = Execute ozone fs -ls -R o3fs://bucket1.fstest/testdir1/
Should contain ${result} localdir1/LOCAL.txt Should contain ${result} localdir1/LOCAL.txt
Should contain ${result} testdir1/NOTICE.txt Should contain ${result} testdir1/NOTICE.txt
Execute ozone fs -mkdir -p o3://bucket2.fstest/testdir2 Execute ozone fs -mkdir -p o3fs://bucket2.fstest/testdir2
Execute ozone fs -mkdir -p o3://bucket3.fstest2/testdir3 Execute ozone fs -mkdir -p o3fs://bucket3.fstest2/testdir3
Execute ozone fs -cp o3://bucket1.fstest/testdir1/localdir1 o3://bucket2.fstest/testdir2/ Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket2.fstest/testdir2/
Execute ozone fs -cp o3://bucket1.fstest/testdir1/localdir1 o3://bucket3.fstest2/testdir3/ Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket3.fstest2/testdir3/
Execute ozone sh key put o3://ozoneManager/fstest/bucket1/KEY.txt NOTICE.txt Execute ozone sh key put o3://ozoneManager/fstest/bucket1/KEY.txt NOTICE.txt
${result} = Execute ozone fs -ls o3://bucket1.fstest/KEY.txt ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/KEY.txt
Should contain ${result} KEY.txt Should contain ${result} KEY.txt
${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3://bucket1.fstest/KEY.txt ${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/KEY.txt
Should Be Equal As Integers ${rc} 1 Should Be Equal As Integers ${rc} 1
Should contain ${result} File exists Should contain ${result} File exists
Execute rm -Rf GET.txt Execute rm -Rf GET.txt
Execute ozone fs -get o3://bucket1.fstest/KEY.txt GET.txt Execute ozone fs -get o3fs://bucket1.fstest/KEY.txt GET.txt
Execute ls -l GET.txt Execute ls -l GET.txt
${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3://abcde.pqrs/ ${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3fs://abcde.pqrs/
Should Be Equal As Integers ${rc} 1 Should Be Equal As Integers ${rc} 1
Should contain ${result} VOLUME_NOT_FOUND Should contain ${result} VOLUME_NOT_FOUND

6
hadoop-ozone/docs/content/OzoneFS.md
View File

@ -42,16 +42,16 @@ Please add the following entry to the core-site.xml.
{{< highlight xml >}} {{< highlight xml >}}
<property> <property>
<name>fs.o3.impl</name> <name>fs.o3fs.impl</name>
<value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value> <value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
</property> </property>
<property> <property>
<name>fs.defaultFS</name> <name>fs.defaultFS</name>
<value>o3://localhost:9864/volume/bucket</value> <value>o3fs://localhost:9864/volume/bucket</value>
</property> </property>
{{< /highlight >}} {{< /highlight >}}
This will make this bucket to be the default file system for HDFS dfs commands and register the o3 file system type.. This will make this bucket to be the default file system for HDFS dfs commands and register the o3fs file system type..
You also need to add the ozone-filesystem.jar file to the classpath: You also need to add the ozone-filesystem.jar file to the classpath:

6
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
View File

@ -35,7 +35,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.rest.OzoneException; import org.apache.hadoop.ozone.client.rest.OzoneException;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_HTTP_SCHEME; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_HTTP_SCHEME;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RPC_SCHEME;
import org.apache.http.client.utils.URIBuilder; import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -83,7 +83,7 @@ public abstract class Handler implements Callable<Void> {
Configuration conf = new OzoneConfiguration(); Configuration conf = new OzoneConfiguration();
String scheme = ozoneURI.getScheme(); String scheme = ozoneURI.getScheme();
if (ozoneURI.getScheme() == null || scheme.isEmpty()) { if (ozoneURI.getScheme() == null || scheme.isEmpty()) {
scheme = OZONE_URI_SCHEME; scheme = OZONE_RPC_SCHEME;
} }
if (scheme.equals(OZONE_HTTP_SCHEME)) { if (scheme.equals(OZONE_HTTP_SCHEME)) {
if (ozoneURI.getHost() != null) { if (ozoneURI.getHost() != null) {
@ -96,7 +96,7 @@ public abstract class Handler implements Callable<Void> {
} else { } else {
client = OzoneClientFactory.getRestClient(conf); client = OzoneClientFactory.getRestClient(conf);
} }
} else if (scheme.equals(OZONE_URI_SCHEME)) { } else if (scheme.equals(OZONE_RPC_SCHEME)) {
if (ozoneURI.getHost() != null) { if (ozoneURI.getHost() != null) {
if (ozoneURI.getPort() == -1) { if (ozoneURI.getPort() == -1) {
client = OzoneClientFactory.getRpcClient(ozoneURI.getHost()); client = OzoneClientFactory.getRpcClient(ozoneURI.getHost());

2
hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
View File

@ -107,7 +107,7 @@ public class OzoneFileSystem extends FileSystem {
if (!matcher.matches()) { if (!matcher.matches()) {
throw new IllegalArgumentException("Ozone file system url should be " throw new IllegalArgumentException("Ozone file system url should be "
+ "in the form o3://bucket.volume"); + "in the form o3fs://bucket.volume");
} }
String bucketStr = matcher.group(1); String bucketStr = matcher.group(1);
String volumeStr = matcher.group(2); String volumeStr = matcher.group(2);

2
hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
View File

@ -231,7 +231,7 @@ public class TestOzoneFileInterfaces {
ozoneFs.pathToKey(new Path("key1/key2"))); ozoneFs.pathToKey(new Path("key1/key2")));
assertEquals("key1/key2", assertEquals("key1/key2",
ozoneFs.pathToKey(new Path("o3://test1/key1/key2"))); ozoneFs.pathToKey(new Path("o3fs://test1/key1/key2")));
} }
private String getCurrentUser() { private String getCurrentUser() {