From 3c11dec714b193525ba5e6190407f2156110d71a Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Wed, 17 Oct 2018 16:46:06 -0700 Subject: [PATCH] HDDS-683. Add a shell command to provide ozone mapping for a S3Bucket. Contributed by Bharat Viswanadham. (cherry picked from commit 4c4364ddd06927fa990d3e66d4eadbd4a7eb9474) --- .../hadoop/ozone/ozShell/TestOzoneShell.java | 42 ++++++++ .../hadoop/ozone/web/ozShell/Shell.java | 5 +- .../web/ozShell/bucket/BucketCommands.java | 3 +- .../web/ozShell/bucket/S3BucketMapping.java | 99 +++++++++++++++++++ 4 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java index 1576d41f33d..c736f81296b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java @@ -1076,6 +1076,48 @@ public class TestOzoneShell { executeWithError(shell, args, "the length should be a positive number"); } + @Test + public void testS3BucketMapping() throws IOException { + String s3Bucket = "bucket1"; + String commandOutput; + createS3Bucket("ozone", s3Bucket); + String volumeName = client.getOzoneVolumeName(s3Bucket); + String[] args = new String[] {"bucket", "path", url + "/" + s3Bucket}; + if (url.startsWith("o3")) { + execute(shell, args); + commandOutput = out.toString(); + assertTrue(commandOutput.contains("Volume name for S3Bucket is : " + + volumeName)); + assertTrue(commandOutput.contains(OzoneConsts.OZONE_URI_SCHEME +"://" + + s3Bucket + "." + volumeName)); + out.reset(); + //Trying to get map for an unknown bucket + args = new String[] {"bucket", "path", url + "/" + "unknownbucket"}; + executeWithError(shell, args, "S3_BUCKET_NOT_FOUND"); + } else { + executeWithError(shell, args, "Ozone REST protocol does not support " + + "this operation"); + } + + // No bucket name + args = new String[] {"bucket", "path", url}; + executeWithError(shell, args, "S3Bucket name is required"); + + // Invalid bucket name + args = new String[] {"bucket", "path", url + "/" + s3Bucket + + "/multipleslash"}; + executeWithError(shell, args, "Invalid S3Bucket name. Delimiters (/) not" + + " allowed"); + } + + private void createS3Bucket(String userName, String s3Bucket) { + try { + client.createS3Bucket("ozone", s3Bucket); + } catch (IOException ex) { + GenericTestUtils.assertExceptionContains("S3_BUCKET_ALREADY_EXISTS", ex); + } + } + private OzoneVolume creatVolume() throws OzoneException, IOException { String volumeName = RandomStringUtils.randomNumeric(5) + "volume"; VolumeArgs volumeArgs = VolumeArgs.newBuilder() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java index 0f3969f32c5..28ab90033ee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java @@ -51,7 +51,7 @@ public class Shell extends GenericCli { public static final String OZONE_URI_DESCRIPTION = "Ozone URI could start " + "with o3:// or http(s):// or without prefix. REST protocol will " + "be used for http(s), RPC otherwise. URI may contain the host and port " - + "of the SCM server. Both are optional. " + + "of the OM server. Both are optional. " + "If they are not specified it will be identified from " + "the config files."; @@ -64,6 +64,9 @@ public class Shell extends GenericCli { public static final String OZONE_KEY_URI_DESCRIPTION = "URI of the volume/bucket/key.\n" + OZONE_URI_DESCRIPTION; + public static final String OZONE_S3BUCKET_URI_DESCRIPTION = "URI of the " + + "S3Bucket.\n" + OZONE_URI_DESCRIPTION; + // General options public static final int DEFAULT_OZONE_PORT = 50070; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java index 870f4d71986..617643ce141 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java @@ -38,7 +38,8 @@ import picocli.CommandLine.ParentCommand; ListBucketHandler.class, CreateBucketHandler.class, UpdateBucketHandler.class, - DeleteBucketHandler.class + DeleteBucketHandler.class, + S3BucketMapping.class }, mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java new file mode 100644 index 00000000000..21ae7467a82 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/S3BucketMapping.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.web.ozShell.bucket; + +import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.client.OzoneClientException; +import org.apache.hadoop.ozone.web.ozShell.Handler; +import org.apache.hadoop.ozone.web.ozShell.Shell; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Parameters; + +/** + * S3Bucket mapping handler, which returns volume name and Ozone fs uri for + * that bucket. + */ +@Command(name = "path", + description = "Returns the ozone path for S3Bucket") +public class S3BucketMapping extends Handler { + + @Parameters(arity = "1..1", description = Shell + .OZONE_S3BUCKET_URI_DESCRIPTION) + private String uri; + + /** + * Executes create bucket. + */ + @Override + public Void call() throws Exception { + + URI ozoneURI = verifyURI(uri); + Path path = Paths.get(ozoneURI.getPath()); + int pathNameCount = path.getNameCount(); + String errorMessage; + + // When just uri is given as http://om:9874, we are getting pathCount + // still as 1, as getPath() is returning empty string. + // So for safer side check, whether it is an empty string + if (pathNameCount == 1) { + String s3Bucket = path.getName(0).toString(); + if (StringUtils.isBlank(s3Bucket)) { + errorMessage = "S3Bucket name is required to get volume name and " + + "Ozone fs Uri"; + throw new OzoneClientException(errorMessage); + } + } + if (pathNameCount != 1) { + if (pathNameCount < 1) { + errorMessage = "S3Bucket name is required to get volume name and " + + "Ozone fs Uri"; + } else { + errorMessage = "Invalid S3Bucket name. Delimiters (/) not allowed in " + + "S3Bucket name"; + } + throw new OzoneClientException(errorMessage); + } + + String s3Bucket = path.getName(0).toString(); + if (isVerbose()) { + System.out.printf("S3Bucket Name : %s%n", s3Bucket); + } + + String mapping = client.getObjectStore().getOzoneBucketMapping(s3Bucket); + String volumeName = client.getObjectStore().getOzoneVolumeName(s3Bucket); + + if (isVerbose()) { + System.out.printf("Mapping created for S3Bucket is : %s%n", mapping); + } + + System.out.printf("Volume name for S3Bucket is : %s%n", volumeName); + + String ozoneFsUri = String.format("%s://%s.%s", OzoneConsts + .OZONE_URI_SCHEME, s3Bucket, volumeName); + + System.out.printf("Ozone FileSystem Uri is : %s%n", ozoneFsUri); + + return null; + } +}