From 989a3929a92edb000cfa486146987fb75a9eda61 Mon Sep 17 00:00:00 2001 From: Aaron Fabbri Date: Mon, 23 Apr 2018 15:26:53 -0700 Subject: [PATCH] HADOOP-14756 S3Guard: expose capability query in MetadataStore and add tests of authoritative mode (Gabor Bota) --- .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 2 + .../fs/s3a/s3guard/LocalMetadataStore.java | 2 + .../hadoop/fs/s3a/s3guard/MetadataStore.java | 2 +- .../s3guard/MetadataStoreCapabilities.java | 43 +++++++++++++++++++ .../fs/s3a/s3guard/MetadataStoreTestBase.java | 31 ++++++++++++- 5 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreCapabilities.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java index c579b3ca3f3..b942ed73496 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java @@ -1154,6 +1154,8 @@ public Map getDiagnostics() throws IOException { map.put(READ_CAPACITY, throughput.getReadCapacityUnits().toString()); map.put(WRITE_CAPACITY, throughput.getWriteCapacityUnits().toString()); map.put(TABLE, desc.toString()); + map.put(MetadataStoreCapabilities.PERSISTS_AUTHORITATIVE_BIT, + Boolean.toString(false)); } else { map.put("name", "DynamoDB Metadata Store"); map.put(TABLE, "none"); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java index 86059c8fa76..306428f9572 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java @@ -447,6 +447,8 @@ public Map getDiagnostics() throws IOException { map.put("name", "local://metadata"); map.put("uriHost", uriHost); map.put("description", "Local in-VM metadata store for testing"); + map.put(MetadataStoreCapabilities.PERSISTS_AUTHORITATIVE_BIT, + Boolean.toString(true)); return map; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java index cdde10307ce..46f8dd3e51c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java @@ -238,7 +238,7 @@ void prune(long modTime, String keyPrefix) /** * Get any diagnostics information from a store, as a list of (key, value) * tuples for display. Arbitrary values; no guarantee of stability. - * These are for debugging only. + * These are for debugging and testing only. * @return a map of strings. * @throws IOException if there is an error */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreCapabilities.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreCapabilities.java new file mode 100644 index 00000000000..c1464402246 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreCapabilities.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.s3guard; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * All the capability constants used for the + * {@link MetadataStore} implementations. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public final class MetadataStoreCapabilities { + + private MetadataStoreCapabilities(){ + } + + /** + * This capability tells if the metadata store supports authoritative + * directories. Used in {@link MetadataStore#getDiagnostics()} as a key + * for this capability. The value can be boolean true or false. + * If the Map.get() returns null for this key, that is interpreted as false. + */ + public static final String PERSISTS_AUTHORITATIVE_BIT = + "persist.authoritative.bit"; +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java index e463ce4706b..478c2e8ad31 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.Set; +import java.util.Map; import com.google.common.collect.Sets; import org.junit.After; @@ -503,7 +504,6 @@ public void testListChildren() throws Exception { assertListingsEqual(dirMeta.getListing(), "/a1/b1", "/a1/b2"); } - // TODO HADOOP-14756 instrument MetadataStore for asserting & testing dirMeta = ms.listChildren(strToPath("/a1/b1")); if (!allowMissing() || dirMeta != null) { assertListingsEqual(dirMeta.getListing(), "/a1/b1/file1", "/a1/b1/file2", @@ -511,6 +511,35 @@ public void testListChildren() throws Exception { } } + private boolean isMetadataStoreAuthoritative() throws IOException { + Map diags = ms.getDiagnostics(); + String isAuth = + diags.get(MetadataStoreCapabilities.PERSISTS_AUTHORITATIVE_BIT); + if(isAuth == null){ + return false; + } + return Boolean.valueOf(isAuth); + } + + @Test + public void testListChildrenAuthoritative() throws IOException { + Assume.assumeTrue("MetadataStore should be capable for authoritative " + + "storage of directories to run this test.", + isMetadataStoreAuthoritative()); + + setupListStatus(); + + DirListingMetadata dirMeta = ms.listChildren(strToPath("/a1/b1")); + dirMeta.setAuthoritative(true); + dirMeta.put(makeFileStatus("/a1/b1/file_new", 100)); + ms.put(dirMeta); + + dirMeta = ms.listChildren(strToPath("/a1/b1")); + assertListingsEqual(dirMeta.getListing(), "/a1/b1/file1", "/a1/b1/file2", + "/a1/b1/c1", "/a1/b1/file_new"); + assertTrue(dirMeta.isAuthoritative()); + } + @Test public void testDirListingRoot() throws Exception { commonTestPutListStatus("/");