Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-18017 Reduce frequency of setStoragePolicy failure warnings

This commit is contained in:
Andrew Purtell 2017-05-08 15:05:59 -07:00
parent 951b23a44c
commit c38bf12444
1 changed files with 44 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
View File

@ -152,6 +152,9 @@ public abstract class FSUtils {
setStoragePolicy(fs, path, storagePolicy); setStoragePolicy(fs, path, storagePolicy);
} }
private static final Map<FileSystem, Boolean> warningMap =
new ConcurrentHashMap<FileSystem, Boolean>();
/** /**
* Sets storage policy for given path. * Sets storage policy for given path.
* If the passed path is a directory, we'll set the storage policy for all files * If the passed path is a directory, we'll set the storage policy for all files
@ -187,17 +190,20 @@ public abstract class FSUtils {
try { try {
distributed = isDistributedFileSystem(fs); distributed = isDistributedFileSystem(fs);
} catch (IOException ioe) { } catch (IOException ioe) {
// This should NEVER happen. if (!warningMap.containsKey(fs)) {
LOG.warn("Failed setStoragePolicy=" + trimmedStoragePolicy + " on path=" + warningMap.put(fs, true);
path + "; failed isDFS test", ioe); LOG.warn("FileSystem isn't an instance of DistributedFileSystem; presuming it doesn't "
+ "support setStoragePolicy. Unable to set storagePolicy=" + trimmedStoragePolicy
+ " on path=" + path);
} else if (LOG.isDebugEnabled()) {
LOG.debug("FileSystem isn't an instance of DistributedFileSystem; presuming it doesn't "
+ "support setStoragePolicy. Unable to set storagePolicy=" + trimmedStoragePolicy
+ " on path=" + path);
}
return; return;
} }
if (distributed) { if (distributed) {
invokeSetStoragePolicy(fs, path, trimmedStoragePolicy); invokeSetStoragePolicy(fs, path, trimmedStoragePolicy);
} else {
LOG.info("FileSystem isn't an instance of DistributedFileSystem; presuming it doesn't " +
"support setStoragePolicy. Unable to set storagePolicy=" + trimmedStoragePolicy +
" on path=" + path);
} }
} }
@ -209,41 +215,56 @@ public abstract class FSUtils {
Method m = null; Method m = null;
try { try {
m = fs.getClass().getDeclaredMethod("setStoragePolicy", m = fs.getClass().getDeclaredMethod("setStoragePolicy",
new Class<?>[] { Path.class, String.class }); new Class<?>[] { Path.class, String.class });
m.setAccessible(true); m.setAccessible(true);
} catch (NoSuchMethodException e) { } catch (NoSuchMethodException e) {
LOG.info("FileSystem doesn't support setStoragePolicy; HDFS-6584 not available " final String msg = "FileSystem doesn't support setStoragePolicy; HDFS-6584 not available";
+ "(hadoop-2.6.0+): " + e.getMessage()); if (!warningMap.containsKey(fs)) {
warningMap.put(fs, true);
LOG.warn(msg, e);
} else if (LOG.isDebugEnabled()) {
LOG.debug(msg, e);
}
m = null;
} catch (SecurityException e) { } catch (SecurityException e) {
LOG.info("Don't have access to setStoragePolicy on FileSystems; HDFS-6584 not available " final String msg = "No access to setStoragePolicy on FileSystem; HDFS-6584 not available";
+ "(hadoop-2.6.0+): ", e); if (!warningMap.containsKey(fs)) {
warningMap.put(fs, true);
LOG.warn(msg, e);
} else if (LOG.isDebugEnabled()) {
LOG.debug(msg, e);
}
m = null; // could happen on setAccessible() m = null; // could happen on setAccessible()
} }
if (m != null) { if (m != null) {
try { try {
m.invoke(fs, path, storagePolicy); m.invoke(fs, path, storagePolicy);
LOG.info("Set storagePolicy=" + storagePolicy + " for path=" + path); if (LOG.isDebugEnabled()) {
LOG.debug("Set storagePolicy=" + storagePolicy + " for path=" + path);
}
} catch (Exception e) { } catch (Exception e) {
// This swallows FNFE, should we be throwing it? seems more likely to indicate dev
// misuse than a runtime problem with HDFS.
if (!warningMap.containsKey(fs)) {
warningMap.put(fs, true);
LOG.warn("Unable to set storagePolicy=" + storagePolicy + " for path=" + path, e);
} else if (LOG.isDebugEnabled()) {
LOG.debug("Unable to set storagePolicy=" + storagePolicy + " for path=" + path, e);
}
// check for lack of HDFS-7228 // check for lack of HDFS-7228
boolean probablyBadPolicy = false;
if (e instanceof InvocationTargetException) { if (e instanceof InvocationTargetException) {
final Throwable exception = e.getCause(); final Throwable exception = e.getCause();
if (exception instanceof RemoteException && if (exception instanceof RemoteException &&
HadoopIllegalArgumentException.class.getName().equals( HadoopIllegalArgumentException.class.getName().equals(
((RemoteException)exception).getClassName())) { ((RemoteException)exception).getClassName())) {
LOG.warn("Given storage policy, '" + storagePolicy + "', was rejected and probably " + if (LOG.isDebugEnabled()) {
LOG.debug("Given storage policy, '" +storagePolicy +"', was rejected and probably " +
"isn't a valid policy for the version of Hadoop you're running. I.e. if you're " + "isn't a valid policy for the version of Hadoop you're running. I.e. if you're " +
"trying to use SSD related policies then you're likely missing HDFS-7228. For " + "trying to use SSD related policies then you're likely missing HDFS-7228. For " +
"more information see the 'ArchivalStorage' docs for your Hadoop release."); "more information see the 'ArchivalStorage' docs for your Hadoop release.");
LOG.debug("More information about the invalid storage policy.", exception); }
probablyBadPolicy = true;
} }
} }
if (!probablyBadPolicy) {
// This swallows FNFE, should we be throwing it? seems more likely to indicate dev
// misuse than a runtime problem with HDFS.
LOG.warn("Unable to set storagePolicy=" + storagePolicy + " for path=" + path, e);
}
} }
} }
} }