HDDS-1542. Create Radix tree to support ozone prefix ACLs. Contributed by Xiaoyu Yao.

This commit is contained in:
Xiaoyu Yao 2019-05-29 13:39:27 -07:00 committed by Anu Engineer
parent 751f0df710
commit 0ead2090a6
4 changed files with 423 additions and 0 deletions

View File

@ -0,0 +1,59 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.util;
import java.util.HashMap;
/**
* Wrapper class for Radix tree node representing Ozone prefix path segment
* separated by "/".
*/
public class RadixNode<T> {
public RadixNode(String name) {
this.name = name;
this.children = new HashMap<>();
}
public String getName() {
return name;
}
public boolean hasChildren() {
return children.isEmpty();
}
public HashMap<String, RadixNode> getChildren() {
return children;
}
public void setValue(T v) {
this.value = v;
}
public T getValue() {
return value;
}
private HashMap<String, RadixNode> children;
private String name;
// TODO: k/v pairs for more metadata as needed
private T value;
}

View File

@ -0,0 +1,214 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.util;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.ozone.OzoneConsts;
import java.util.ArrayList;
import java.util.HashMap;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
/**
* Wrapper class for handling Ozone prefix path lookup of ACL APIs
* with radix tree.
*/
public class RadixTree<T> {
/**
* create a empty radix tree with root only.
*/
public RadixTree() {
root = new RadixNode<T>(PATH_DELIMITER);
}
/**
* If the Radix tree contains root only.
* @return true if the radix tree contains root only.
*/
public boolean isEmpty() {
return root.hasChildren();
}
/**
* Insert prefix tree node without value, value can be ACL or other metadata
* of the prefix path.
* @param path
*/
public void insert(String path) {
insert(path, null);
}
/**
* Insert prefix tree node with value, value can be ACL or other metadata
* of the prefix path.
* @param path
* @param val
*/
public void insert(String path, T val) {
// all prefix path inserted should end with "/"
RadixNode<T> n = root;
Path p = Paths.get(path);
for (int level = 0; level < p.getNameCount(); level++) {
HashMap<String, RadixNode> child = n.getChildren();
String component = p.getName(level).toString();
if (child.containsKey(component)) {
n = child.get(component);
} else {
RadixNode tmp = new RadixNode(component);
child.put(component, tmp);
n = tmp;
}
}
if (val != null) {
n.setValue(val);
}
}
/**
* Get the last node in the exact prefix path that matches in the tree.
* @param path - prefix path
* @return last node in the prefix tree or null if non exact prefix matchl
*/
public RadixNode<T> getLastNodeInPrefixPath(String path) {
List<RadixNode<T>> lpp = getLongestPrefixPath(path);
Path p = Paths.get(path);
if (lpp.size() != p.getNameCount() + 1) {
return null;
} else {
return lpp.get(p.getNameCount());
}
}
/**
* Remove prefix path.
* @param path
*/
public void removePrefixPath(String path) {
Path p = Paths.get(path);
removePrefixPathInternal(root, p, 0);
}
/**
* Recursively remove non-overlapped part of the prefix path from radix tree.
* @param current current radix tree node.
* @param path prefix path to be removed.
* @param level current recursive level.
* @return true if current radix node can be removed.
* (not overlapped with other path),
* false otherwise.
*/
private boolean removePrefixPathInternal(RadixNode<T> current,
Path path, int level) {
// last component is processed
if (level == path.getNameCount()) {
return current.hasChildren();
}
// not last component, recur for next component
String name = path.getName(level).toString();
RadixNode<T> node = current.getChildren().get(name);
if (node == null) {
return false;
}
if (removePrefixPathInternal(node, path, level+1)) {
current.getChildren().remove(name);
return current.hasChildren();
}
return false;
}
/**
* Get the longest prefix path.
* @param path - prefix path.
* @return longest prefix path as list of RadixNode.
*/
public List<RadixNode<T>> getLongestPrefixPath(String path) {
RadixNode n = root;
Path p = Paths.get(path);
int level = 0;
List<RadixNode<T>> result = new ArrayList<>();
result.add(root);
while (level < p.getNameCount()) {
HashMap<String, RadixNode> children = n.getChildren();
if (children.isEmpty()) {
break;
}
String component = p.getName(level).toString();
if (children.containsKey(component)) {
n = children.get(component);
result.add(n);
level++;
} else {
break;
}
}
return result;
}
@VisibleForTesting
/**
* Convert radix path to string format for output.
* @param path - radix path represented by list of radix nodes.
* @return radix path as string separated by "/".
* Note: the path will always be normalized with and ending "/".
*/
public static String radixPathToString(List<RadixNode<Integer>> path) {
StringBuilder sb = new StringBuilder();
for (RadixNode n : path) {
sb.append(n.getName());
sb.append(n.getName().equals(PATH_DELIMITER) ? "" : PATH_DELIMITER);
}
return sb.toString();
}
/**
* Get the longest prefix path.
* @param path - prefix path.
* @return longest prefix path as String separated by "/".
*/
public String getLongestPrefix(String path) {
RadixNode<T> n = root;
Path p = Paths.get(path);
int level = 0;
while (level < p.getNameCount()) {
HashMap<String, RadixNode> children = n.getChildren();
if (children.isEmpty()) {
break;
}
String component = p.getName(level).toString();
if (children.containsKey(component)) {
n = children.get(component);
level++;
} else {
break;
}
}
return level >= 1 ?
Paths.get(root.getName()).resolve(p.subpath(0, level)).toString() :
root.getName();
}
// root of a radix tree has a name of "/" and may optionally has it value.
private RadixNode root;
private final static String PATH_DELIMITER = OzoneConsts.OZONE_URI_DELIMITER;
}

View File

@ -0,0 +1,129 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.util;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.List;
import static org.junit.Assert.assertEquals;
/**
* Test Ozone Radix tree operations.
*/
public class TestRadixTree {
final static RadixTree<Integer> ROOT = new RadixTree<>();
@BeforeClass
public static void setupRadixTree() {
// Test prefix paths with an empty tree
assertEquals(true, ROOT.isEmpty());
assertEquals("/", ROOT.getLongestPrefix("/a/b/c"));
assertEquals("/", RadixTree.radixPathToString(
ROOT.getLongestPrefixPath("/a/g")));
// Build Radix tree below for testing.
// a
// |
// b
// / \
// c e
// / \ / \ \
// d f g dir1 dir2(1000)
// |
// g
// |
// h
ROOT.insert("/a/b/c/d");
ROOT.insert("/a/b/c/d/g/h");
ROOT.insert("/a/b/c/f");
ROOT.insert("/a/b/e/g");
ROOT.insert("/a/b/e/dir1");
ROOT.insert("/a/b/e/dir2", 1000);
}
/**
* Tests if insert and build prefix tree is correct.
*/
@Test
public void testGetLongestPrefix() {
assertEquals("/a/b/c", ROOT.getLongestPrefix("/a/b/c"));
assertEquals("/a/b", ROOT.getLongestPrefix("/a/b"));
assertEquals("/a", ROOT.getLongestPrefix("/a"));
assertEquals("/a/b/e/g", ROOT.getLongestPrefix("/a/b/e/g/h"));
assertEquals("/", ROOT.getLongestPrefix("/d/b/c"));
assertEquals("/a/b/e", ROOT.getLongestPrefix("/a/b/e/dir3"));
assertEquals("/a/b/c/d", ROOT.getLongestPrefix("/a/b/c/d/p"));
assertEquals("/a/b/c/f", ROOT.getLongestPrefix("/a/b/c/f/p"));
}
@Test
public void testGetLongestPrefixPath() {
List<RadixNode<Integer>> lpp =
ROOT.getLongestPrefixPath("/a/b/c/d/g/p");
RadixNode<Integer> lpn = lpp.get(lpp.size()-1);
assertEquals("g", lpn.getName());
lpn.setValue(100);
List<RadixNode<Integer>> lpq =
ROOT.getLongestPrefixPath("/a/b/c/d/g/q");
RadixNode<Integer> lqn = lpp.get(lpq.size()-1);
System.out.print(RadixTree.radixPathToString(lpq));
assertEquals(lpn, lqn);
assertEquals("g", lqn.getName());
assertEquals(100, (int)lqn.getValue());
assertEquals("/a/", RadixTree.radixPathToString(
ROOT.getLongestPrefixPath("/a/g")));
}
@Test
public void testGetLastNoeInPrefixPath() {
assertEquals(null, ROOT.getLastNodeInPrefixPath("/a/g"));
RadixNode<Integer> ln = ROOT.getLastNodeInPrefixPath("/a/b/e/dir1");
assertEquals("dir1", ln.getName());
}
@Test
public void testRemovePrefixPath() {
// Remove, test and restore
// Remove partially overlapped path
ROOT.removePrefixPath("/a/b/c/d/g/h");
assertEquals("/a/b/c", ROOT.getLongestPrefix("a/b/c/d"));
ROOT.insert("/a/b/c/d/g/h");
// Remove fully overlapped path
ROOT.removePrefixPath("/a/b/c/d");
assertEquals("/a/b/c/d", ROOT.getLongestPrefix("a/b/c/d"));
ROOT.insert("/a/b/c/d");
// Remove non existing path
ROOT.removePrefixPath("/d/a");
assertEquals("/a/b/c/d", ROOT.getLongestPrefix("a/b/c/d"));
}
}

View File

@ -0,0 +1,21 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.util;
/**
* Unit tests of generic ozone utils.
*/