HADOOP-12738. Create unit test to automatically compare Common related classes and core-default.xml. Contributed by Ray Chiang.
(cherry picked from commit 68b4564e78)
This commit is contained in:
Masatake Iwasaki
parent
7ff4fb89ca
commit
b4ed3285a1
|
|
@ -0,0 +1,176 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.conf;
|
||||
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
|
||||
import org.apache.hadoop.fs.AbstractFileSystem;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||
import org.apache.hadoop.fs.ftp.FtpConfigKeys;
|
||||
import org.apache.hadoop.fs.local.LocalConfigKeys;
|
||||
import org.apache.hadoop.ha.SshFenceByTcpPort;
|
||||
import org.apache.hadoop.ha.ZKFailoverController;
|
||||
import org.apache.hadoop.http.HttpServer2;
|
||||
import org.apache.hadoop.io.nativeio.NativeIO;
|
||||
import org.apache.hadoop.security.CompositeGroupsMapping;
|
||||
import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
|
||||
import org.apache.hadoop.security.LdapGroupsMapping;
|
||||
import org.apache.hadoop.security.http.CrossOriginFilter;
|
||||
import org.apache.hadoop.security.ssl.SSLFactory;
|
||||
|
||||
/**
|
||||
* Unit test class to compare the following Hadoop Configuration classes:
|
||||
* <p></p>
|
||||
* {@link org.apache.hadoop.fs.AbstractFileSystem}
|
||||
* {@link org.apache.hadoop.fs.CommonConfigurationKeys}
|
||||
* {@link org.apache.hadoop.fs.CommonConfigurationKeysPublic}
|
||||
* {@link org.apache.hadoop.fs.ftp.FtpConfigKeys}
|
||||
* {@link org.apache.hadoop.fs.local.LocalConfigKeys}
|
||||
* {@link org.apache.hadoop.ha.SshFenceByTcpPort}
|
||||
* {@link org.apache.hadoop.http.HttpServer2}
|
||||
* {@link org.apache.hadoop.security.LdapGroupsMapping}
|
||||
* {@link org.apache.hadoop.security.http.CrossOriginFilter}
|
||||
* {@link org.apache.hadoop.security.ssl.SSLFactory}
|
||||
* <p></p>
|
||||
* against core-site.xml for missing properties. Currently only
|
||||
* throws an error if the class is missing a property.
|
||||
* <p></p>
|
||||
* Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase}
|
||||
* for how this class works.
|
||||
*/
|
||||
public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Override
|
||||
public void initializeMemberVariables() {
|
||||
xmlFilename = new String("core-default.xml");
|
||||
configurationClasses = new Class[] {
|
||||
CommonConfigurationKeys.class,
|
||||
CommonConfigurationKeysPublic.class,
|
||||
LocalConfigKeys.class,
|
||||
FtpConfigKeys.class,
|
||||
SshFenceByTcpPort.class,
|
||||
LdapGroupsMapping.class,
|
||||
ZKFailoverController.class,
|
||||
SSLFactory.class,
|
||||
CompositeGroupsMapping.class
|
||||
};
|
||||
|
||||
// Initialize used variables
|
||||
xmlPropsToSkipCompare = new HashSet<String>();
|
||||
xmlPrefixToSkipCompare = new HashSet<String>();
|
||||
configurationPropsToSkipCompare = new HashSet<String>();
|
||||
|
||||
// Set error modes
|
||||
errorIfMissingConfigProps = true;
|
||||
errorIfMissingXmlProps = false;
|
||||
|
||||
// Lots of properties not in the above classes
|
||||
xmlPropsToSkipCompare.add("fs.ftp.password.localhost");
|
||||
xmlPropsToSkipCompare.add("fs.ftp.user.localhost");
|
||||
xmlPropsToSkipCompare.add("fs.s3.block.size");
|
||||
xmlPropsToSkipCompare.add("hadoop.tmp.dir");
|
||||
xmlPropsToSkipCompare.add("nfs3.mountd.port");
|
||||
xmlPropsToSkipCompare.add("nfs3.server.port");
|
||||
xmlPropsToSkipCompare.add("test.fs.s3.name");
|
||||
xmlPropsToSkipCompare.add("test.fs.s3n.name");
|
||||
|
||||
// S3/S3A properties are in a different subtree.
|
||||
// - org.apache.hadoop.fs.s3.S3FileSystemConfigKeys
|
||||
// - org.apache.hadoop.fs.s3native.S3NativeFileSystemConfigKeys
|
||||
xmlPrefixToSkipCompare.add("fs.s3.");
|
||||
xmlPrefixToSkipCompare.add("fs.s3a.");
|
||||
xmlPrefixToSkipCompare.add("fs.s3n.");
|
||||
xmlPrefixToSkipCompare.add("s3.");
|
||||
xmlPrefixToSkipCompare.add("s3native.");
|
||||
|
||||
// Deprecated properties. These should eventually be removed from the
|
||||
// class.
|
||||
configurationPropsToSkipCompare
|
||||
.add(CommonConfigurationKeysPublic.IO_SORT_MB_KEY);
|
||||
configurationPropsToSkipCompare
|
||||
.add(CommonConfigurationKeysPublic.IO_SORT_FACTOR_KEY);
|
||||
|
||||
// Irrelevant property
|
||||
configurationPropsToSkipCompare.add("dr.who");
|
||||
|
||||
// XML deprecated properties.
|
||||
xmlPropsToSkipCompare.add("io.seqfile.lazydecompress");
|
||||
xmlPropsToSkipCompare.add("io.seqfile.sorter.recordlimit");
|
||||
// - org.apache.hadoop.hdfs.client.HdfsClientConfigKeys
|
||||
xmlPropsToSkipCompare
|
||||
.add("io.bytes.per.checksum");
|
||||
|
||||
// Properties in other classes that aren't easily determined
|
||||
// (not following naming convention, in a different project, not public,
|
||||
// etc.)
|
||||
// - org.apache.hadoop.http.HttpServer2.FILTER_INITIALIZER_PROPERTY
|
||||
xmlPropsToSkipCompare.add("hadoop.http.filter.initializers");
|
||||
// - org.apache.hadoop.security.HttpCrossOriginFilterInitializer
|
||||
xmlPrefixToSkipCompare.add(HttpCrossOriginFilterInitializer.PREFIX);
|
||||
xmlPrefixToSkipCompare.add("fs.AbstractFileSystem.");
|
||||
// - org.apache.hadoop.ha.SshFenceByTcpPort
|
||||
xmlPrefixToSkipCompare.add("dfs.ha.fencing.ssh.");
|
||||
// - org.apache.hadoop.classification.RegistryConstants
|
||||
xmlPrefixToSkipCompare.add("hadoop.registry.");
|
||||
// - org.apache.hadoop.security.AuthenticationFilterInitializer
|
||||
xmlPrefixToSkipCompare.add("hadoop.http.authentication.");
|
||||
// - org.apache.hadoop.crypto.key.kms.KMSClientProvider;
|
||||
xmlPropsToSkipCompare.add(KMSClientProvider.AUTH_RETRY);
|
||||
// - org.apache.hadoop.io.nativeio.NativeIO
|
||||
xmlPropsToSkipCompare.add("hadoop.workaround.non.threadsafe.getpwuid");
|
||||
// - org.apache.hadoop.hdfs.DFSConfigKeys
|
||||
xmlPropsToSkipCompare.add("dfs.ha.fencing.methods");
|
||||
// - org.apache.hadoop.fs.CommonConfigurationKeysPublic
|
||||
xmlPrefixToSkipCompare
|
||||
.add(CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX);
|
||||
// - org.apache.hadoop.hdfs.server.datanode.DataNode
|
||||
xmlPropsToSkipCompare.add("hadoop.common.configuration.version");
|
||||
// - org.apache.hadoop.fs.FileSystem
|
||||
xmlPropsToSkipCompare.add("fs.har.impl.disable.cache");
|
||||
// - org.apache.hadoop.fs.FileSystem#getFileSystemClass()
|
||||
xmlPropsToSkipCompare.add("fs.swift.impl");
|
||||
// - package org.apache.hadoop.tracing.TraceUtils ?
|
||||
xmlPropsToSkipCompare.add("hadoop.htrace.span.receiver.classes");
|
||||
// Private keys
|
||||
// - org.apache.hadoop.ha.ZKFailoverController;
|
||||
xmlPropsToSkipCompare.add("ha.zookeeper.parent-znode");
|
||||
xmlPropsToSkipCompare.add("ha.zookeeper.session-timeout.ms");
|
||||
// - Where is this used?
|
||||
xmlPrefixToSkipCompare
|
||||
.add(CommonConfigurationKeys.FS_CLIENT_HTRACE_PREFIX);
|
||||
// - org.apache.hadoop.security.UserGroupInformation
|
||||
xmlPropsToSkipCompare.add("hadoop.kerberos.kinit.command");
|
||||
// - org.apache.hadoop.net.NetUtils
|
||||
xmlPropsToSkipCompare
|
||||
.add("hadoop.rpc.socket.factory.class.ClientProtocol");
|
||||
// - Where is this used?
|
||||
xmlPropsToSkipCompare.add("hadoop.ssl.enabled");
|
||||
|
||||
// Keys with no corresponding variable
|
||||
// - org.apache.hadoop.io.compress.bzip2.Bzip2Factory
|
||||
xmlPropsToSkipCompare.add("io.compression.codec.bzip2.library");
|
||||
// - org.apache.hadoop.io.SequenceFile
|
||||
xmlPropsToSkipCompare.add("io.seqfile.local.dir");
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue