Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-14938. Configuration.updatingResource map should be initialized lazily (misha@cloudera.com via rkanter)

This commit is contained in:
Robert Kanter 2017-10-13 13:52:58 -07:00
parent 7a27c2c367
commit e163f41850
1 changed files with 55 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -290,9 +290,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
/** /**
* Stores the mapping of key to the resource which modifies or loads * Stores the mapping of key to the resource which modifies or loads
* the key most recently * the key most recently. Created lazily to avoid wasting memory.
*/ */
private Map<String, String[]> updatingResource; private volatile Map<String, String[]> updatingResource;
/** /**
* Specify exact input factory to avoid time finding correct one. * Specify exact input factory to avoid time finding correct one.
@ -749,7 +749,6 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
*/ */
public Configuration(boolean loadDefaults) { public Configuration(boolean loadDefaults) {
this.loadDefaults = loadDefaults; this.loadDefaults = loadDefaults;
updatingResource = new ConcurrentHashMap<String, String[]>();
// Register all classes holding property tags with // Register all classes holding property tags with
REGISTERED_TAG_CLASS.put("core", CorePropertyTag.class); REGISTERED_TAG_CLASS.put("core", CorePropertyTag.class);
@ -768,25 +767,27 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Configuration(Configuration other) { public Configuration(Configuration other) {
this.resources = (ArrayList<Resource>) other.resources.clone(); this.resources = (ArrayList<Resource>) other.resources.clone();
synchronized(other) { synchronized(other) {
if (other.properties != null) { if (other.properties != null) {
this.properties = (Properties)other.properties.clone(); this.properties = (Properties)other.properties.clone();
} }
if (other.overlay!=null) { if (other.overlay!=null) {
this.overlay = (Properties)other.overlay.clone(); this.overlay = (Properties)other.overlay.clone();
} }
if (other.updatingResource != null) {
this.updatingResource = new ConcurrentHashMap<String, String[]>(
other.updatingResource);
}
this.finalParameters = Collections.newSetFromMap(
new ConcurrentHashMap<String, Boolean>());
this.finalParameters.addAll(other.finalParameters);
this.REGISTERED_TAG_CLASS.putAll(other.REGISTERED_TAG_CLASS);
this.propertyTagsMap.putAll(other.propertyTagsMap);
}
this.updatingResource = new ConcurrentHashMap<String, String[]>(
other.updatingResource);
this.finalParameters = Collections.newSetFromMap(
new ConcurrentHashMap<String, Boolean>());
this.finalParameters.addAll(other.finalParameters);
this.REGISTERED_TAG_CLASS.putAll(other.REGISTERED_TAG_CLASS);
this.propertyTagsMap.putAll(other.propertyTagsMap);
}
synchronized(Configuration.class) { synchronized(Configuration.class) {
REGISTRY.put(this, null); REGISTRY.put(this, null);
} }
@ -1277,14 +1278,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
String newSource = (source == null ? "programmatically" : source); String newSource = (source == null ? "programmatically" : source);
if (!isDeprecated(name)) { if (!isDeprecated(name)) {
updatingResource.put(name, new String[] {newSource}); putIntoUpdatingResource(name, new String[] {newSource});
String[] altNames = getAlternativeNames(name); String[] altNames = getAlternativeNames(name);
if(altNames != null) { if(altNames != null) {
for(String n: altNames) { for(String n: altNames) {
if(!n.equals(name)) { if(!n.equals(name)) {
getOverlay().setProperty(n, value); getOverlay().setProperty(n, value);
getProps().setProperty(n, value); getProps().setProperty(n, value);
updatingResource.put(n, new String[] {newSource}); putIntoUpdatingResource(n, new String[] {newSource});
} }
} }
} }
@ -1295,7 +1296,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
for(String n : names) { for(String n : names) {
getOverlay().setProperty(n, value); getOverlay().setProperty(n, value);
getProps().setProperty(n, value); getProps().setProperty(n, value);
updatingResource.put(n, new String[] {altSource}); putIntoUpdatingResource(n, new String[] {altSource});
} }
} }
} }
@ -2634,17 +2635,19 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
protected synchronized Properties getProps() { protected synchronized Properties getProps() {
if (properties == null) { if (properties == null) {
properties = new Properties(); properties = new Properties();
Map<String, String[]> backup = Map<String, String[]> backup = updatingResource != null ?
new ConcurrentHashMap<String, String[]>(updatingResource); new ConcurrentHashMap<String, String[]>(updatingResource) : null;
loadResources(properties, resources, quietmode); loadResources(properties, resources, quietmode);
if (overlay != null) { if (overlay != null) {
properties.putAll(overlay); properties.putAll(overlay);
for (Map.Entry<Object,Object> item: overlay.entrySet()) { if (backup != null) {
String key = (String)item.getKey(); for (Map.Entry<Object, Object> item : overlay.entrySet()) {
String[] source = backup.get(key); String key = (String) item.getKey();
if(source != null) { String[] source = backup.get(key);
updatingResource.put(key, source); if (source != null) {
updatingResource.put(key, source);
}
} }
} }
} }
@ -3060,8 +3063,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
} }
if (!finalParameters.contains(attr)) { if (!finalParameters.contains(attr)) {
properties.setProperty(attr, value); properties.setProperty(attr, value);
if(source != null) { if (source != null) {
updatingResource.put(attr, source); putIntoUpdatingResource(attr, source);
} }
} else { } else {
// This is a final parameter so check for overrides. // This is a final parameter so check for overrides.
@ -3366,9 +3369,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
redactor.redact(name, config.get(name))); redactor.redact(name, config.get(name)));
jsonGen.writeBooleanField("isFinal", jsonGen.writeBooleanField("isFinal",
config.finalParameters.contains(name)); config.finalParameters.contains(name));
String[] resources = config.updatingResource.get(name); String[] resources = config.updatingResource != null ?
config.updatingResource.get(name) : null;
String resource = UNKNOWN_RESOURCE; String resource = UNKNOWN_RESOURCE;
if(resources != null && resources.length > 0) { if (resources != null && resources.length > 0) {
resource = resources[0]; resource = resources[0];
} }
jsonGen.writeStringField("resource", resource); jsonGen.writeStringField("resource", resource);
@ -3448,8 +3452,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
String value = org.apache.hadoop.io.Text.readString(in); String value = org.apache.hadoop.io.Text.readString(in);
set(key, value); set(key, value);
String sources[] = WritableUtils.readCompressedStringArray(in); String sources[] = WritableUtils.readCompressedStringArray(in);
if(sources != null) { if (sources != null) {
updatingResource.put(key, sources); putIntoUpdatingResource(key, sources);
} }
} }
} }
@ -3462,8 +3466,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
for(Map.Entry<Object, Object> item: props.entrySet()) { for(Map.Entry<Object, Object> item: props.entrySet()) {
org.apache.hadoop.io.Text.writeString(out, (String) item.getKey()); org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
org.apache.hadoop.io.Text.writeString(out, (String) item.getValue()); org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
WritableUtils.writeCompressedStringArray(out, WritableUtils.writeCompressedStringArray(out, updatingResource != null ?
updatingResource.get(item.getKey())); updatingResource.get(item.getKey()) : null);
} }
} }
@ -3563,4 +3567,17 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
} }
return tag; return tag;
} }
private void putIntoUpdatingResource(String key, String[] value) {
Map<String, String[]> localUR = updatingResource;
if (localUR == null) {
synchronized (this) {
localUR = updatingResource;
if (localUR == null) {
updatingResource = localUR = new ConcurrentHashMap<>(8);
}
}
}
localUR.put(key, value);
}
} }