Merge -r 1304596:1304597 from trunk to branch-0.23. Fixes: HADOOP-7030
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1304599 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas White
parent
1f3e5ebc0b
commit
cb7add5de2
|
|
@ -22,6 +22,9 @@ Release 0.23.3 - UNRELEASED
|
|||
HADOOP-8121. Active Directory Group Mapping Service. (Jonathan Natkins via
|
||||
atm)
|
||||
|
||||
HADOOP-7030. Add TableMapping topology implementation to read host to rack
|
||||
mapping from a file. (Patrick Angeles and tomwhite via tomwhite)
|
||||
|
||||
IMPROVEMENTS
|
||||
|
||||
HADOOP-7524. Change RPC to allow multiple protocols including multuple
|
||||
|
|
|
|||
|
|
@ -63,6 +63,10 @@ public class CommonConfigurationKeysPublic {
|
|||
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
|
||||
public static final String NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY =
|
||||
"net.topology.node.switch.mapping.impl";
|
||||
|
||||
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
|
||||
public static final String NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY =
|
||||
"net.topology.table.file.name";
|
||||
|
||||
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
|
||||
public static final String FS_TRASH_CHECKPOINT_INTERVAL_KEY =
|
||||
|
|
|
|||
|
|
@ -0,0 +1,147 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.net;
|
||||
|
||||
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.conf.Configured;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Simple {@link DNSToSwitchMapping} implementation that reads a 2 column text
|
||||
* file. The columns are separated by whitespace. The first column is a DNS or
|
||||
* IP address and the second column specifies the rack where the address maps.
|
||||
* </p>
|
||||
* <p>
|
||||
* This class uses the configuration parameter {@code
|
||||
* net.topology.table.file.name} to locate the mapping file.
|
||||
* </p>
|
||||
* <p>
|
||||
* Calls to {@link #resolve(List)} will look up the address as defined in the
|
||||
* mapping file. If no entry corresponding to the address is found, the value
|
||||
* {@code /default-rack} is returned.
|
||||
* </p>
|
||||
*/
|
||||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Evolving
|
||||
public class TableMapping extends CachedDNSToSwitchMapping {
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(TableMapping.class);
|
||||
|
||||
public TableMapping() {
|
||||
super(new RawTableMapping());
|
||||
}
|
||||
|
||||
private RawTableMapping getRawMapping() {
|
||||
return (RawTableMapping) rawMapping;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Configuration getConf() {
|
||||
return getRawMapping().getConf();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setConf(Configuration conf) {
|
||||
super.setConf(conf);
|
||||
getRawMapping().setConf(conf);
|
||||
}
|
||||
|
||||
private static final class RawTableMapping extends Configured
|
||||
implements DNSToSwitchMapping {
|
||||
|
||||
private final Map<String, String> map = new HashMap<String, String>();
|
||||
private boolean initialized = false;
|
||||
|
||||
private synchronized void load() {
|
||||
map.clear();
|
||||
|
||||
String filename = getConf().get(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, null);
|
||||
if (StringUtils.isBlank(filename)) {
|
||||
LOG.warn(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY + " not configured. "
|
||||
+ NetworkTopology.DEFAULT_RACK + " will be returned.");
|
||||
return;
|
||||
}
|
||||
|
||||
BufferedReader reader = null;
|
||||
try {
|
||||
reader = new BufferedReader(new FileReader(filename));
|
||||
String line = reader.readLine();
|
||||
while (line != null) {
|
||||
line = line.trim();
|
||||
if (line.length() != 0 && line.charAt(0) != '#') {
|
||||
String[] columns = line.split("\\s+");
|
||||
if (columns.length == 2) {
|
||||
map.put(columns[0], columns[1]);
|
||||
} else {
|
||||
LOG.warn("Line does not have two columns. Ignoring. " + line);
|
||||
}
|
||||
}
|
||||
line = reader.readLine();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.warn(filename + " cannot be read. " + NetworkTopology.DEFAULT_RACK
|
||||
+ " will be returned.", e);
|
||||
map.clear();
|
||||
} finally {
|
||||
if (reader != null) {
|
||||
try {
|
||||
reader.close();
|
||||
} catch (IOException e) {
|
||||
LOG.warn(filename + " cannot be read. "
|
||||
+ NetworkTopology.DEFAULT_RACK + " will be returned.", e);
|
||||
map.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized List<String> resolve(List<String> names) {
|
||||
if (!initialized) {
|
||||
initialized = true;
|
||||
load();
|
||||
}
|
||||
|
||||
List<String> results = new ArrayList<String>(names.size());
|
||||
for (String name : names) {
|
||||
String result = map.get(name);
|
||||
if (result != null) {
|
||||
results.add(result);
|
||||
} else {
|
||||
results.add(NetworkTopology.DEFAULT_RACK);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -688,6 +688,19 @@
|
|||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>net.topology.table.file.name</name>
|
||||
<value></value>
|
||||
<description> The file name for a topology file, which is used when the
|
||||
net.topology.script.file.name property is set to
|
||||
org.apache.hadoop.net.TableMapping. The file format is a two column text
|
||||
file, with columns separated by whitespace. The first column is a DNS or
|
||||
IP address and the second column specifies the rack where the address maps.
|
||||
If no entry corresponding to a host in the cluster is found, then
|
||||
/default-rack is assumed.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<!-- Local file system -->
|
||||
<property>
|
||||
<name>file.stream-buffer-size</name>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,145 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.net;
|
||||
|
||||
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import com.google.common.base.Charsets;
|
||||
import com.google.common.io.Files;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestTableMapping {
|
||||
|
||||
private File mappingFile;
|
||||
|
||||
@Before
|
||||
public void setUp() throws IOException {
|
||||
mappingFile = File.createTempFile(getClass().getSimpleName(), ".txt");
|
||||
Files.write("a.b.c /rack1\n" +
|
||||
"1.2.3\t/rack2\n", mappingFile, Charsets.UTF_8);
|
||||
mappingFile.deleteOnExit();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testResolve() throws IOException {
|
||||
TableMapping mapping = new TableMapping();
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, mappingFile.getCanonicalPath());
|
||||
mapping.setConf(conf);
|
||||
|
||||
List<String> names = new ArrayList<String>();
|
||||
names.add("a.b.c");
|
||||
names.add("1.2.3");
|
||||
|
||||
List<String> result = mapping.resolve(names);
|
||||
assertEquals(names.size(), result.size());
|
||||
assertEquals("/rack1", result.get(0));
|
||||
assertEquals("/rack2", result.get(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTableCaching() throws IOException {
|
||||
TableMapping mapping = new TableMapping();
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, mappingFile.getCanonicalPath());
|
||||
mapping.setConf(conf);
|
||||
|
||||
List<String> names = new ArrayList<String>();
|
||||
names.add("a.b.c");
|
||||
names.add("1.2.3");
|
||||
|
||||
List<String> result1 = mapping.resolve(names);
|
||||
assertEquals(names.size(), result1.size());
|
||||
assertEquals("/rack1", result1.get(0));
|
||||
assertEquals("/rack2", result1.get(1));
|
||||
|
||||
// unset the file, see if it gets read again
|
||||
conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, "some bad value for a file");
|
||||
|
||||
List<String> result2 = mapping.resolve(names);
|
||||
assertEquals(result1, result2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoFile() {
|
||||
TableMapping mapping = new TableMapping();
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
mapping.setConf(conf);
|
||||
|
||||
List<String> names = new ArrayList<String>();
|
||||
names.add("a.b.c");
|
||||
names.add("1.2.3");
|
||||
|
||||
List<String> result = mapping.resolve(names);
|
||||
assertEquals(names.size(), result.size());
|
||||
assertEquals(NetworkTopology.DEFAULT_RACK, result.get(0));
|
||||
assertEquals(NetworkTopology.DEFAULT_RACK, result.get(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFileDoesNotExist() {
|
||||
TableMapping mapping = new TableMapping();
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, "/this/file/does/not/exist");
|
||||
mapping.setConf(conf);
|
||||
|
||||
List<String> names = new ArrayList<String>();
|
||||
names.add("a.b.c");
|
||||
names.add("1.2.3");
|
||||
|
||||
List<String> result = mapping.resolve(names);
|
||||
assertEquals(names.size(), result.size());
|
||||
assertEquals(result.get(0), NetworkTopology.DEFAULT_RACK);
|
||||
assertEquals(result.get(1), NetworkTopology.DEFAULT_RACK);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBadFile() throws IOException {
|
||||
Files.write("bad contents", mappingFile, Charsets.UTF_8);
|
||||
|
||||
TableMapping mapping = new TableMapping();
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, mappingFile.getCanonicalPath());
|
||||
mapping.setConf(conf);
|
||||
|
||||
List<String> names = new ArrayList<String>();
|
||||
names.add("a.b.c");
|
||||
names.add("1.2.3");
|
||||
|
||||
List<String> result = mapping.resolve(names);
|
||||
assertEquals(names.size(), result.size());
|
||||
assertEquals(result.get(0), NetworkTopology.DEFAULT_RACK);
|
||||
assertEquals(result.get(1), NetworkTopology.DEFAULT_RACK);
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue