From d65b957776c4f055f82549a610fd1e8494580fe6 Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Wed, 5 Oct 2016 12:16:49 -0700 Subject: [PATCH] YARN-5702. Refactor TestPBImplRecords for reuse in other YARN modules. Contributed by Subru Krishnan --- .../yarn/api/BasePBImplRecordsTest.java | 264 ++++++++++++++++++ .../hadoop/yarn/api/TestPBImplRecords.java | 257 +---------------- 2 files changed, 270 insertions(+), 251 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java new file mode 100644 index 00000000000..21a737dd557 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java @@ -0,0 +1,264 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Assert; + +import java.lang.reflect.*; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * Generic helper class to validate protocol records. + */ +public class BasePBImplRecordsTest { + static final Log LOG = LogFactory.getLog(BasePBImplRecordsTest.class); + + @SuppressWarnings("checkstyle:visibilitymodifier") + protected static HashMap typeValueCache = + new HashMap(); + private static Random rand = new Random(); + private static byte [] bytes = new byte[] {'1', '2', '3', '4'}; + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static Object genTypeValue(Type type) { + Object ret = typeValueCache.get(type); + if (ret != null) { + return ret; + } + // only use positive primitive values + if (type.equals(boolean.class)) { + return rand.nextBoolean(); + } else if (type.equals(byte.class)) { + return bytes[rand.nextInt(4)]; + } else if (type.equals(int.class) || type.equals(Integer.class)) { + return rand.nextInt(1000000); + } else if (type.equals(long.class) || type.equals(Long.class)) { + return Long.valueOf(rand.nextInt(1000000)); + } else if (type.equals(float.class)) { + return rand.nextFloat(); + } else if (type.equals(double.class)) { + return rand.nextDouble(); + } else if (type.equals(String.class)) { + return String.format("%c%c%c", + 'a' + rand.nextInt(26), + 'a' + rand.nextInt(26), + 'a' + rand.nextInt(26)); + } else if (type instanceof Class) { + Class clazz = (Class)type; + if (clazz.isArray()) { + Class compClass = clazz.getComponentType(); + if (compClass != null) { + ret = Array.newInstance(compClass, 2); + Array.set(ret, 0, genTypeValue(compClass)); + Array.set(ret, 1, genTypeValue(compClass)); + } + } else if (clazz.isEnum()) { + Object [] values = clazz.getEnumConstants(); + ret = values[rand.nextInt(values.length)]; + } else if (clazz.equals(ByteBuffer.class)) { + // return new ByteBuffer every time + // to prevent potential side effects + ByteBuffer buff = ByteBuffer.allocate(4); + rand.nextBytes(buff.array()); + return buff; + } + } else if (type instanceof ParameterizedType) { + ParameterizedType pt = (ParameterizedType)type; + Type rawType = pt.getRawType(); + Type [] params = pt.getActualTypeArguments(); + // only support EnumSet, List, Set, Map + if (rawType.equals(EnumSet.class)) { + if (params[0] instanceof Class) { + Class c = (Class)(params[0]); + return EnumSet.allOf(c); + } + } if (rawType.equals(List.class)) { + ret = Lists.newArrayList(genTypeValue(params[0])); + } else if (rawType.equals(Set.class)) { + ret = Sets.newHashSet(genTypeValue(params[0])); + } else if (rawType.equals(Map.class)) { + Map map = Maps.newHashMap(); + map.put(genTypeValue(params[0]), genTypeValue(params[1])); + ret = map; + } + } + if (ret == null) { + throw new IllegalArgumentException("type " + type + " is not supported"); + } + typeValueCache.put(type, ret); + return ret; + } + + /** + * this method generate record instance by calling newIntance + * using reflection, add register the generated value to typeValueCache + */ + @SuppressWarnings("rawtypes") + protected static Object generateByNewInstance(Class clazz) throws Exception { + Object ret = typeValueCache.get(clazz); + if (ret != null) { + return ret; + } + Method newInstance = null; + Type [] paramTypes = new Type[0]; + // get newInstance method with most parameters + for (Method m : clazz.getMethods()) { + int mod = m.getModifiers(); + if (m.getDeclaringClass().equals(clazz) && + Modifier.isPublic(mod) && + Modifier.isStatic(mod) && + m.getName().equals("newInstance")) { + Type [] pts = m.getGenericParameterTypes(); + if (newInstance == null + || (pts.length > paramTypes.length)) { + newInstance = m; + paramTypes = pts; + } + } + } + if (newInstance == null) { + throw new IllegalArgumentException("type " + clazz.getName() + + " does not have newInstance method"); + } + Object [] args = new Object[paramTypes.length]; + for (int i=0;i Map getGetSetPairs(Class recordClass) + throws Exception { + Map ret = new HashMap(); + Method [] methods = recordClass.getDeclaredMethods(); + // get all get methods + for (int i = 0; i < methods.length; i++) { + Method m = methods[i]; + int mod = m.getModifiers(); + if (m.getDeclaringClass().equals(recordClass) && + Modifier.isPublic(mod) && + (!Modifier.isStatic(mod))) { + String name = m.getName(); + if (name.equals("getProto")) { + continue; + } + if ((name.length() > 3) && name.startsWith("get") && + (m.getParameterTypes().length == 0)) { + String propertyName = name.substring(3); + Type valueType = m.getGenericReturnType(); + GetSetPair p = ret.get(propertyName); + if (p == null) { + p = new GetSetPair(); + p.propertyName = propertyName; + p.type = valueType; + p.getMethod = m; + ret.put(propertyName, p); + } else { + Assert.fail("Multiple get method with same name: " + recordClass + + p.propertyName); + } + } + } + } + // match get methods with set methods + for (int i = 0; i < methods.length; i++) { + Method m = methods[i]; + int mod = m.getModifiers(); + if (m.getDeclaringClass().equals(recordClass) && + Modifier.isPublic(mod) && + (!Modifier.isStatic(mod))) { + String name = m.getName(); + if (name.startsWith("set") && (m.getParameterTypes().length == 1)) { + String propertyName = name.substring(3); + Type valueType = m.getGenericParameterTypes()[0]; + GetSetPair p = ret.get(propertyName); + if (p != null && p.type.equals(valueType)) { + p.setMethod = m; + } + } + } + } + // exclude incomplete get/set pair, and generate test value + Iterator> itr = ret.entrySet().iterator(); + while (itr.hasNext()) { + Map.Entry cur = itr.next(); + GetSetPair gsp = cur.getValue(); + if ((gsp.getMethod == null) || + (gsp.setMethod == null)) { + LOG.info(String.format("Exclude protential property: %s\n", gsp.propertyName)); + itr.remove(); + } else { + LOG.info(String.format("New property: %s type: %s", gsp.toString(), gsp.type)); + gsp.testValue = genTypeValue(gsp.type); + LOG.info(String.format(" testValue: %s\n", gsp.testValue)); + } + } + return ret; + } + + protected void validatePBImplRecord(Class recordClass, + Class

protoClass) + throws Exception { + LOG.info(String.format("Validate %s %s\n", recordClass.getName(), + protoClass.getName())); + Constructor emptyConstructor = recordClass.getConstructor(); + Constructor pbConstructor = recordClass.getConstructor(protoClass); + Method getProto = recordClass.getDeclaredMethod("getProto"); + Map getSetPairs = getGetSetPairs(recordClass); + R origRecord = emptyConstructor.newInstance(); + for (GetSetPair gsp : getSetPairs.values()) { + gsp.setMethod.invoke(origRecord, gsp.testValue); + } + Object ret = getProto.invoke(origRecord); + Assert.assertNotNull(recordClass.getName() + "#getProto returns null", ret); + if (!(protoClass.isAssignableFrom(ret.getClass()))) { + Assert.fail("Illegal getProto method return type: " + ret.getClass()); + } + R deserRecord = pbConstructor.newInstance(ret); + Assert.assertEquals("whole " + recordClass + " records should be equal", + origRecord, deserRecord); + for (GetSetPair gsp : getSetPairs.values()) { + Object origValue = gsp.getMethod.invoke(origRecord); + Object deserValue = gsp.getMethod.invoke(deserRecord); + Assert.assertEquals("property " + recordClass.getName() + "#" + + gsp.propertyName + " should be equal", origValue, deserValue); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 527048650c4..4b71282ffe0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -16,26 +16,8 @@ * limitations under the License. */ package org.apache.hadoop.yarn.api; -import java.io.IOException; -import java.lang.reflect.Array; -import java.lang.reflect.Constructor; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.nio.ByteBuffer; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.Set; - +import com.google.common.collect.ImmutableSet; import org.apache.commons.lang.math.LongRange; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto; import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto; @@ -343,127 +325,12 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import java.io.IOException; -public class TestPBImplRecords { - static final Log LOG = LogFactory.getLog(TestPBImplRecords.class); - - private static HashMap typeValueCache = new HashMap(); - private static Random rand = new Random(); - private static byte [] bytes = new byte[] {'1', '2', '3', '4'}; - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static Object genTypeValue(Type type) { - Object ret = typeValueCache.get(type); - if (ret != null) { - return ret; - } - // only use positive primitive values - if (type.equals(boolean.class)) { - return rand.nextBoolean(); - } else if (type.equals(byte.class)) { - return bytes[rand.nextInt(4)]; - } else if (type.equals(int.class) || type.equals(Integer.class)) { - return rand.nextInt(1000000); - } else if (type.equals(long.class) || type.equals(Long.class)) { - return Long.valueOf(rand.nextInt(1000000)); - } else if (type.equals(float.class)) { - return rand.nextFloat(); - } else if (type.equals(double.class)) { - return rand.nextDouble(); - } else if (type.equals(String.class)) { - return String.format("%c%c%c", - 'a' + rand.nextInt(26), - 'a' + rand.nextInt(26), - 'a' + rand.nextInt(26)); - } else if (type instanceof Class) { - Class clazz = (Class)type; - if (clazz.isArray()) { - Class compClass = clazz.getComponentType(); - if (compClass != null) { - ret = Array.newInstance(compClass, 2); - Array.set(ret, 0, genTypeValue(compClass)); - Array.set(ret, 1, genTypeValue(compClass)); - } - } else if (clazz.isEnum()) { - Object [] values = clazz.getEnumConstants(); - ret = values[rand.nextInt(values.length)]; - } else if (clazz.equals(ByteBuffer.class)) { - // return new ByteBuffer every time - // to prevent potential side effects - ByteBuffer buff = ByteBuffer.allocate(4); - rand.nextBytes(buff.array()); - return buff; - } - } else if (type instanceof ParameterizedType) { - ParameterizedType pt = (ParameterizedType)type; - Type rawType = pt.getRawType(); - Type [] params = pt.getActualTypeArguments(); - // only support EnumSet, List, Set, Map - if (rawType.equals(EnumSet.class)) { - if (params[0] instanceof Class) { - Class c = (Class)(params[0]); - return EnumSet.allOf(c); - } - } if (rawType.equals(List.class)) { - ret = Lists.newArrayList(genTypeValue(params[0])); - } else if (rawType.equals(Set.class)) { - ret = Sets.newHashSet(genTypeValue(params[0])); - } else if (rawType.equals(Map.class)) { - Map map = Maps.newHashMap(); - map.put(genTypeValue(params[0]), genTypeValue(params[1])); - ret = map; - } - } - if (ret == null) { - throw new IllegalArgumentException("type " + type + " is not supported"); - } - typeValueCache.put(type, ret); - return ret; - } - - /** - * this method generate record instance by calling newIntance - * using reflection, add register the generated value to typeValueCache - */ - @SuppressWarnings("rawtypes") - private static Object generateByNewInstance(Class clazz) throws Exception { - Object ret = typeValueCache.get(clazz); - if (ret != null) { - return ret; - } - Method newInstance = null; - Type [] paramTypes = new Type[0]; - // get newInstance method with most parameters - for (Method m : clazz.getMethods()) { - int mod = m.getModifiers(); - if (m.getDeclaringClass().equals(clazz) && - Modifier.isPublic(mod) && - Modifier.isStatic(mod) && - m.getName().equals("newInstance")) { - Type [] pts = m.getGenericParameterTypes(); - if (newInstance == null - || (pts.length > paramTypes.length)) { - newInstance = m; - paramTypes = pts; - } - } - } - if (newInstance == null) { - throw new IllegalArgumentException("type " + clazz.getName() + - " does not have newInstance method"); - } - Object [] args = new Object[paramTypes.length]; - for (int i=0;i Map getGetSetPairs(Class recordClass) - throws Exception { - Map ret = new HashMap(); - Method [] methods = recordClass.getDeclaredMethods(); - // get all get methods - for (int i = 0; i < methods.length; i++) { - Method m = methods[i]; - int mod = m.getModifiers(); - if (m.getDeclaringClass().equals(recordClass) && - Modifier.isPublic(mod) && - (!Modifier.isStatic(mod))) { - String name = m.getName(); - if (name.equals("getProto")) { - continue; - } - if ((name.length() > 3) && name.startsWith("get") && - (m.getParameterTypes().length == 0)) { - String propertyName = name.substring(3); - Type valueType = m.getGenericReturnType(); - GetSetPair p = ret.get(propertyName); - if (p == null) { - p = new GetSetPair(); - p.propertyName = propertyName; - p.type = valueType; - p.getMethod = m; - ret.put(propertyName, p); - } else { - Assert.fail("Multiple get method with same name: " + recordClass - + p.propertyName); - } - } - } - } - // match get methods with set methods - for (int i = 0; i < methods.length; i++) { - Method m = methods[i]; - int mod = m.getModifiers(); - if (m.getDeclaringClass().equals(recordClass) && - Modifier.isPublic(mod) && - (!Modifier.isStatic(mod))) { - String name = m.getName(); - if (name.startsWith("set") && (m.getParameterTypes().length == 1)) { - String propertyName = name.substring(3); - Type valueType = m.getGenericParameterTypes()[0]; - GetSetPair p = ret.get(propertyName); - if (p != null && p.type.equals(valueType)) { - p.setMethod = m; - } - } - } - } - // exclude incomplete get/set pair, and generate test value - Iterator> itr = ret.entrySet().iterator(); - while (itr.hasNext()) { - Entry cur = itr.next(); - GetSetPair gsp = cur.getValue(); - if ((gsp.getMethod == null) || - (gsp.setMethod == null)) { - LOG.info(String.format("Exclude protential property: %s\n", gsp.propertyName)); - itr.remove(); - } else { - LOG.info(String.format("New property: %s type: %s", gsp.toString(), gsp.type)); - gsp.testValue = genTypeValue(gsp.type); - LOG.info(String.format(" testValue: %s\n", gsp.testValue)); - } - } - return ret; - } - - private void validatePBImplRecord(Class recordClass, - Class

protoClass) - throws Exception { - LOG.info(String.format("Validate %s %s\n", recordClass.getName(), - protoClass.getName())); - Constructor emptyConstructor = recordClass.getConstructor(); - Constructor pbConstructor = recordClass.getConstructor(protoClass); - Method getProto = recordClass.getDeclaredMethod("getProto"); - Map getSetPairs = getGetSetPairs(recordClass); - R origRecord = emptyConstructor.newInstance(); - for (GetSetPair gsp : getSetPairs.values()) { - gsp.setMethod.invoke(origRecord, gsp.testValue); - } - Object ret = getProto.invoke(origRecord); - Assert.assertNotNull(recordClass.getName() + "#getProto returns null", ret); - if (!(protoClass.isAssignableFrom(ret.getClass()))) { - Assert.fail("Illegal getProto method return type: " + ret.getClass()); - } - R deserRecord = pbConstructor.newInstance(ret); - Assert.assertEquals("whole " + recordClass + " records should be equal", - origRecord, deserRecord); - for (GetSetPair gsp : getSetPairs.values()) { - Object origValue = gsp.getMethod.invoke(origRecord); - Object deserValue = gsp.getMethod.invoke(deserRecord); - Assert.assertEquals("property " + recordClass.getName() + "#" - + gsp.propertyName + " should be equal", origValue, deserValue); - } - } - @Test public void testAllocateRequestPBImpl() throws Exception { validatePBImplRecord(AllocateRequestPBImpl.class, AllocateRequestProto.class);