diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java index 6febcaf2ac2..44712d5b9c9 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretManagerImpl.java @@ -61,8 +61,7 @@ public class S3SecretManagerImpl implements S3SecretManager { public S3SecretValue getS3Secret(String kerberosID) throws IOException { Preconditions.checkArgument(Strings.isNotBlank(kerberosID), "kerberosID cannot be null or empty."); - String awsAccessKeyStr = DigestUtils.md5Hex(kerberosID); - byte[] awsAccessKey = awsAccessKeyStr.getBytes(UTF_8); + byte[] awsAccessKey = kerberosID.getBytes(UTF_8); S3SecretValue result = null; omMetadataManager.getLock().acquireS3SecretLock(kerberosID); try { @@ -77,33 +76,31 @@ public class S3SecretManagerImpl implements S3SecretManager { result = S3SecretValue.fromProtobuf( OzoneManagerProtocolProtos.S3Secret.parseFrom(s3Secret)); } - result.setAwsAccessKey(awsAccessKeyStr); } finally { omMetadataManager.getLock().releaseS3SecretLock(kerberosID); } - LOG.trace("Secret for kerberosID:{},accessKey:{}, proto:{}", kerberosID, - awsAccessKeyStr, result); + LOG.trace("Secret for accessKey:{}, proto:{}", kerberosID, result); return result; } @Override - public String getS3UserSecretString(String awsAccessKeyId) + public String getS3UserSecretString(String kerberosID) throws IOException { - Preconditions.checkArgument(Strings.isNotBlank(awsAccessKeyId), + Preconditions.checkArgument(Strings.isNotBlank(kerberosID), "awsAccessKeyId cannot be null or empty."); - LOG.trace("Get secret for awsAccessKey:{}", awsAccessKeyId); + LOG.trace("Get secret for awsAccessKey:{}", kerberosID); byte[] s3Secret; - omMetadataManager.getLock().acquireS3SecretLock(awsAccessKeyId); + omMetadataManager.getLock().acquireS3SecretLock(kerberosID); try { s3Secret = omMetadataManager.getS3SecretTable() - .get(awsAccessKeyId.getBytes(UTF_8)); + .get(kerberosID.getBytes(UTF_8)); if (s3Secret == null) { throw new OzoneSecurityException("S3 secret not found for " + - "awsAccessKeyId " + awsAccessKeyId, S3_SECRET_NOT_FOUND); + "awsAccessKeyId " + kerberosID, S3_SECRET_NOT_FOUND); } } finally { - omMetadataManager.getLock().releaseS3SecretLock(awsAccessKeyId); + omMetadataManager.getLock().releaseS3SecretLock(kerberosID); } return OzoneManagerProtocolProtos.S3Secret.parseFrom(s3Secret) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java index 23f4c059752..2608e779062 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.ozone.om.helpers; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; /** @@ -26,12 +25,10 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; public class S3SecretValue { private String kerberosID; private String awsSecret; - private String awsAccessKey; public S3SecretValue(String kerberosID, String awsSecret) { this.kerberosID = kerberosID; this.awsSecret = awsSecret; - this.awsAccessKey = DigestUtils.md5Hex(kerberosID); } public String getKerberosID() { @@ -51,11 +48,7 @@ public class S3SecretValue { } public String getAwsAccessKey() { - return awsAccessKey; - } - - public void setAwsAccessKey(String awsAccessKey) { - this.awsAccessKey = awsAccessKey; + return kerberosID; } public static S3SecretValue fromProtobuf( @@ -72,6 +65,6 @@ public class S3SecretValue { @Override public String toString() { - return "awsAccessKey=" + awsAccessKey + "\nawsSecret=" + awsSecret; + return "awsAccessKey=" + kerberosID + "\nawsSecret=" + awsSecret; } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index 58a29ed507f..bd496d0803f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -81,6 +81,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; +import org.apache.hadoop.ozone.s3.util.OzoneS3Util; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.util.Time; @@ -92,6 +93,8 @@ import org.apache.commons.lang3.RandomUtils; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.either; import org.junit.Assert; + +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; @@ -288,6 +291,23 @@ public abstract class TestOzoneRpcClientAbstract { Assert.assertTrue(volume.getCreationTime() >= currentTime); } + @Test + public void testCreateSecureS3Bucket() throws IOException { + long currentTime = Time.now(); + String userName = "ozone/localhost@EXAMPLE.COM"; + String bucketName = UUID.randomUUID().toString(); + String s3VolumeName = OzoneS3Util.getVolumeName(userName); + store.createS3Bucket(s3VolumeName, bucketName); + String volumeName = store.getOzoneVolumeName(bucketName); + assertEquals(volumeName, "s3" + s3VolumeName); + + OzoneVolume volume = store.getVolume(volumeName); + OzoneBucket bucket = volume.getBucket(bucketName); + Assert.assertEquals(bucketName, bucket.getName()); + Assert.assertTrue(bucket.getCreationTime() >= currentTime); + Assert.assertTrue(volume.getCreationTime() >= currentTime); + } + @Test public void testListS3Buckets() diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index b9a29a6f2c5..b85b8d3c0ef 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -53,6 +53,8 @@ import org.apache.hadoop.ozone.s3.util.S3StorageType; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.lang3.StringUtils; + +import static org.apache.hadoop.ozone.s3.util.OzoneS3Util.getVolumeName; import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE; import org.apache.http.HttpStatus; import org.slf4j.Logger; @@ -196,9 +198,10 @@ public class BucketEndpoint extends EndpointBase { public Response put(@PathParam("bucket") String bucketName, @Context HttpHeaders httpHeaders) throws IOException, OS3Exception { - String userName = getAuthenticationHeaderParser().getAccessKeyID(); + String volumeName = getVolumeName(getAuthenticationHeaderParser(). + getAccessKeyID()); - String location = createS3Bucket(userName, bucketName); + String location = createS3Bucket(volumeName, bucketName); LOG.info("Location is {}", location); return Response.status(HttpStatus.SC_OK).header("Location", location) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java index bb918408da3..23d02e9f2b9 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java @@ -34,6 +34,8 @@ import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.ozone.s3.util.OzoneS3Util.getVolumeName; + /** * Top level rest endpoint. */ @@ -63,8 +65,9 @@ public class RootEndpoint extends EndpointBase { .header("Location", "/static/") .build(); } - String userName = authenticationHeaderParser.getAccessKeyID(); - Iterator bucketIterator = listS3Buckets(userName, + String volumeName = getVolumeName(authenticationHeaderParser. + getAccessKeyID()); + Iterator bucketIterator = listS3Buckets(volumeName, null); while (bucketIterator.hasNext()) { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/Credential.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/Credential.java index b53d41d618d..883980af7ec 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/Credential.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/Credential.java @@ -59,13 +59,25 @@ public class Credential { @SuppressWarnings("StringSplitter") public void parseCredential() throws OS3Exception { String[] split = credential.split("/"); - if (split.length == 5) { + switch (split.length) { + case 5: + // Ex: dkjad922329ddnks/20190321/us-west-1/s3/aws4_request accessKeyID = split[0].trim(); date = split[1].trim(); awsRegion = split[2].trim(); awsService = split[3].trim(); awsRequest = split[4].trim(); - } else { + return; + case 6: + // Access id is kerberos principal. + // Ex: testuser/om@EXAMPLE.COM/20190321/us-west-1/s3/aws4_request + accessKeyID = split[0] + "/" +split[1]; + date = split[2].trim(); + awsRegion = split[3].trim(); + awsService = split[4].trim(); + awsRequest = split[5].trim(); + return; + default: LOG.error("Credentials not in expected format. credential:{}", credential); throw S3ErrorTable.newError(S3ErrorTable.MALFORMED_HEADER, credential); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java new file mode 100644 index 00000000000..129ea2d293d --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.s3.util; + +import org.apache.commons.codec.digest.DigestUtils; +import java.util.Objects; + +/** + * Ozone util for S3 related operations. + */ +public final class OzoneS3Util { + + private OzoneS3Util() { + } + + public static String getVolumeName(String userName) { + Objects.requireNonNull(userName); + return DigestUtils.md5Hex(userName); + } +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java index 0636eaf33c8..b7512cb370d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser; import static org.junit.Assert.assertEquals; +import org.apache.hadoop.ozone.s3.util.OzoneS3Util; import org.junit.Before; import org.junit.Test; @@ -61,10 +62,11 @@ public class TestRootList { ListBucketResponse response = (ListBucketResponse) rootEndpoint.get().getEntity(); assertEquals(0, response.getBucketsNum()); + String volumeName = OzoneS3Util.getVolumeName(userName); String bucketBaseName = "bucket-" + getClass().getName(); for(int i = 0; i < 10; i++) { - objectStoreStub.createS3Bucket(userName, bucketBaseName + i); + objectStoreStub.createS3Bucket(volumeName, bucketBaseName + i); } response = (ListBucketResponse) rootEndpoint.get().getEntity(); assertEquals(10, response.getBucketsNum());