Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-10659. Namenode crashes after Journalnode re-installation in an HA cluster due to missing paxos directory. Contributed by star, Hanisha Koneru. 

Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
This commit is contained in:
Hanisha Koneru 2019-06-10 13:43:56 -07:00 committed by Wei-Chiu Chuang
parent e94e643584
commit 7217494f40
2 changed files with 18 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
View File

@ -82,7 +82,8 @@ class JNStorage extends Storage {
} }
@Override @Override
public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException { public boolean isPreUpgradableLayout(StorageDirectory sd)
throws IOException {
return false; return false;
} }
@ -90,7 +91,8 @@ class JNStorage extends Storage {
* Find an edits file spanning the given transaction ID range. * Find an edits file spanning the given transaction ID range.
* If no such file exists, an exception is thrown. * If no such file exists, an exception is thrown.
*/ */
File findFinalizedEditsFile(long startTxId, long endTxId) throws IOException { File findFinalizedEditsFile(long startTxId, long endTxId)
throws IOException {
File ret = new File(sd.getCurrentDir(), File ret = new File(sd.getCurrentDir(),
NNStorage.getFinalizedEditsFileName(startTxId, endTxId)); NNStorage.getFinalizedEditsFileName(startTxId, endTxId));
if (!ret.exists()) { if (!ret.exists()) {
@ -152,11 +154,18 @@ class JNStorage extends Storage {
* paxos-like recovery process for the given log segment. * paxos-like recovery process for the given log segment.
*/ */
File getPaxosFile(long segmentTxId) { File getPaxosFile(long segmentTxId) {
return new File(getPaxosDir(), String.valueOf(segmentTxId)); return new File(getOrCreatePaxosDir(), String.valueOf(segmentTxId));
} }
File getPaxosDir() { File getOrCreatePaxosDir() {
return new File(sd.getCurrentDir(), "paxos"); File paxosDir = new File(sd.getCurrentDir(), "paxos");
if(!paxosDir.exists()) {
LOG.info("Creating paxos dir: {}", paxosDir.toPath());
if(!paxosDir.mkdir()) {
LOG.error("Could not create paxos dir: {}", paxosDir.toPath());
}
}
return paxosDir;
} }
File getRoot() { File getRoot() {
@ -170,7 +179,8 @@ class JNStorage extends Storage {
void purgeDataOlderThan(long minTxIdToKeep) throws IOException { void purgeDataOlderThan(long minTxIdToKeep) throws IOException {
purgeMatching(sd.getCurrentDir(), purgeMatching(sd.getCurrentDir(),
CURRENT_DIR_PURGE_REGEXES, minTxIdToKeep); CURRENT_DIR_PURGE_REGEXES, minTxIdToKeep);
purgeMatching(getPaxosDir(), PAXOS_DIR_PURGE_REGEXES, minTxIdToKeep); purgeMatching(getOrCreatePaxosDir(),
PAXOS_DIR_PURGE_REGEXES, minTxIdToKeep);
} }
/** /**
@ -221,16 +231,10 @@ class JNStorage extends Storage {
unlockAll(); unlockAll();
sd.clearDirectory(); sd.clearDirectory();
writeProperties(sd); writeProperties(sd);
createPaxosDir(); getOrCreatePaxosDir();
analyzeStorage(); analyzeStorage();
} }
void createPaxosDir() throws IOException {
if (!getPaxosDir().mkdirs()) {
throw new IOException("Could not create paxos dir: " + getPaxosDir());
}
}
void analyzeStorage() throws IOException { void analyzeStorage() throws IOException {
this.state = sd.analyzeStorage(StartupOption.REGULAR, this); this.state = sd.analyzeStorage(StartupOption.REGULAR, this);
if (state == StorageState.NORMAL) { if (state == StorageState.NORMAL) {

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
View File

@ -1119,7 +1119,7 @@ public class Journal implements Closeable {
+ ".\n new LV = " + storage.getLayoutVersion() + ".\n new LV = " + storage.getLayoutVersion()
+ "; new CTime = " + storage.getCTime()); + "; new CTime = " + storage.getCTime());
storage.getJournalManager().doUpgrade(storage); storage.getJournalManager().doUpgrade(storage);
storage.createPaxosDir(); storage.getOrCreatePaxosDir();
// Copy over the contents of the epoch data files to the new dir. // Copy over the contents of the epoch data files to the new dir.
File currentDir = storage.getSingularStorageDir().getCurrentDir(); File currentDir = storage.getSingularStorageDir().getCurrentDir();