HDFS-2863. Failures observed if dfs.edits.dir and shared.edits.dir have same directories. Contributed by Bikas Saha.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1240267 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Aaron Myers 2012-02-03 18:04:21 +00:00
parent d2b5531b0f
commit 969318cfc2
4 changed files with 84 additions and 1 deletions

View File

@ -158,3 +158,5 @@ HDFS-2860. TestDFSRollback#testRollback is failing. (atm)
HDFS-2769. HA: When HA is enabled with a shared edits dir, that dir should be
marked required. (atm via eli)
HDFS-2863. Failures observed if dfs.edits.dir and shared.edits.dir have same directories. (Bikas Saha via atm)

View File

@ -190,6 +190,7 @@ import org.apache.hadoop.util.VersionInfo;
import org.mortbay.util.ajax.JSON;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
@ -680,6 +681,19 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
public static Collection<URI> getNamespaceEditsDirs(Configuration conf) {
Collection<URI> editsDirs = getStorageDirs(conf, DFS_NAMENODE_EDITS_DIR_KEY);
editsDirs.addAll(getSharedEditsDirs(conf));
Set<URI> uniqueEditsDirs = new HashSet<URI>();
uniqueEditsDirs.addAll(editsDirs);
if (uniqueEditsDirs.size() != editsDirs.size()) {
// clearing and re-initializing editsDirs to preserve Collection semantics
// assigning finalEditsDirs to editsDirs would leak Set semantics in the
// return value and cause unexpected results downstream. eg future addAll
// calls. Perf is not an issue since these are small lists.
editsDirs.clear();
editsDirs.addAll(uniqueEditsDirs);
LOG.warn("Overlapping entries in " + DFS_NAMENODE_EDITS_DIR_KEY
+ " and/or " + DFS_NAMENODE_SHARED_EDITS_DIR_KEY
+ ". Using the following entries: " + Joiner.on(',').join(editsDirs));
}
if (editsDirs.isEmpty()) {
// If this is the case, no edit dirs have been explicitly configured.
// Image dirs are to be used for edits too.

View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.junit.Assert.*;
import java.net.URI;
import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
import org.junit.Test;
public class TestFSNamesystem {
/**
* Tests that the namenode edits dirs are gotten with duplicates removed
*/
@Test
public void testUniqueEditDirs() {
Configuration config = new Configuration();
config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "
+ "file://edits/dir1,file://edits/dir1"); // overlapping internally
// getNamespaceEditsDirs removes duplicates
Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(config);
assertEquals(2, editsDirs.size());
}
}

View File

@ -17,8 +17,11 @@
*/
package org.apache.hadoop.hdfs.server.namenode.ha;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import java.net.URI;
import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@ -78,4 +81,21 @@ public class TestHAConfiguration {
assertEquals(HOST_B + ":" + DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT,
checkpointer.getActiveNNAddress());
}
/**
* Tests that the namenode edits dirs and shared edits dirs are gotten with
* duplicates removed
*/
@Test
public void testHAUniqueEditDirs() {
Configuration config = new Configuration();
config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "
+ "file://edits/shared/dir"); // overlapping
config.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, "file://edits/shared/dir");
// getNamespaceEditsDirs removes duplicates across edits and shared.edits
Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(config);
assertEquals(2, editsDirs.size());
}
}