HDFS-2863. Failures observed if dfs.edits.dir and shared.edits.dir have same directories. Contributed by Bikas Saha.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1240267 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
d2b5531b0f
commit
969318cfc2
|
@ -158,3 +158,5 @@ HDFS-2860. TestDFSRollback#testRollback is failing. (atm)
|
|||
|
||||
HDFS-2769. HA: When HA is enabled with a shared edits dir, that dir should be
|
||||
marked required. (atm via eli)
|
||||
|
||||
HDFS-2863. Failures observed if dfs.edits.dir and shared.edits.dir have same directories. (Bikas Saha via atm)
|
||||
|
|
|
@ -190,6 +190,7 @@ import org.apache.hadoop.util.VersionInfo;
|
|||
import org.mortbay.util.ajax.JSON;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
@ -680,6 +681,19 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
|||
public static Collection<URI> getNamespaceEditsDirs(Configuration conf) {
|
||||
Collection<URI> editsDirs = getStorageDirs(conf, DFS_NAMENODE_EDITS_DIR_KEY);
|
||||
editsDirs.addAll(getSharedEditsDirs(conf));
|
||||
Set<URI> uniqueEditsDirs = new HashSet<URI>();
|
||||
uniqueEditsDirs.addAll(editsDirs);
|
||||
if (uniqueEditsDirs.size() != editsDirs.size()) {
|
||||
// clearing and re-initializing editsDirs to preserve Collection semantics
|
||||
// assigning finalEditsDirs to editsDirs would leak Set semantics in the
|
||||
// return value and cause unexpected results downstream. eg future addAll
|
||||
// calls. Perf is not an issue since these are small lists.
|
||||
editsDirs.clear();
|
||||
editsDirs.addAll(uniqueEditsDirs);
|
||||
LOG.warn("Overlapping entries in " + DFS_NAMENODE_EDITS_DIR_KEY
|
||||
+ " and/or " + DFS_NAMENODE_SHARED_EDITS_DIR_KEY
|
||||
+ ". Using the following entries: " + Joiner.on(',').join(editsDirs));
|
||||
}
|
||||
if (editsDirs.isEmpty()) {
|
||||
// If this is the case, no edit dirs have been explicitly configured.
|
||||
// Image dirs are to be used for edits too.
|
||||
|
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hdfs.server.namenode;
|
||||
|
||||
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.Collection;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestFSNamesystem {
|
||||
|
||||
/**
|
||||
* Tests that the namenode edits dirs are gotten with duplicates removed
|
||||
*/
|
||||
@Test
|
||||
public void testUniqueEditDirs() {
|
||||
Configuration config = new Configuration();
|
||||
|
||||
config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "
|
||||
+ "file://edits/dir1,file://edits/dir1"); // overlapping internally
|
||||
|
||||
// getNamespaceEditsDirs removes duplicates
|
||||
Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(config);
|
||||
assertEquals(2, editsDirs.size());
|
||||
}
|
||||
|
||||
}
|
|
@ -17,8 +17,11 @@
|
|||
*/
|
||||
package org.apache.hadoop.hdfs.server.namenode.ha;
|
||||
|
||||
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
|
||||
import static org.junit.Assert.*;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.Collection;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
|
@ -78,4 +81,21 @@ public class TestHAConfiguration {
|
|||
assertEquals(HOST_B + ":" + DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT,
|
||||
checkpointer.getActiveNNAddress());
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests that the namenode edits dirs and shared edits dirs are gotten with
|
||||
* duplicates removed
|
||||
*/
|
||||
@Test
|
||||
public void testHAUniqueEditDirs() {
|
||||
Configuration config = new Configuration();
|
||||
|
||||
config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "
|
||||
+ "file://edits/shared/dir"); // overlapping
|
||||
config.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, "file://edits/shared/dir");
|
||||
|
||||
// getNamespaceEditsDirs removes duplicates across edits and shared.edits
|
||||
Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(config);
|
||||
assertEquals(2, editsDirs.size());
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue