Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-15973. Configuration: Included properties are not cached if resource is a stream. Contributed by Eric Payne

This commit is contained in:
Jason Lowe 2018-12-20 10:02:18 -06:00
parent 500b2a0ca6
commit d62bfaf1a4
2 changed files with 103 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -2848,10 +2848,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
// otherwise fallback to a file resource
// xi:include are treated as inline and retain current source
URL include = getResource(confInclude);
Properties tmpProps = new Properties();
if (include != null) {
Resource classpathResource = new Resource(include, name,
wrapper.isParserRestricted());
loadResource(properties, classpathResource, quiet);
loadResource(tmpProps, classpathResource, quiet);
} else {
URL url;
try {
@ -2873,8 +2874,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
Resource uriResource = new Resource(url, name,
wrapper.isParserRestricted());
loadResource(properties, uriResource, quiet);
loadResource(tmpProps, uriResource, quiet);
}
toAddTo.putAll(tmpProps);
break;
case "fallback":
fallbackEntered = true;

99
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -17,10 +17,12 @@
*/
package org.apache.hadoop.conf;
import java.io.BufferedInputStream;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
@ -773,6 +775,103 @@ public class TestConfiguration extends TestCase {
tearDown();
}
// When a resource is parsed as an input stream the first time, included
// properties are saved within the config. However, the included properties
// are not cached in the resource object. So, if an additional resource is
// added after the config is parsed the first time, the config loses the
// prperties that were included from the first resource.
public void testIncludesFromInputStreamWhenResourceAdded() throws Exception {
tearDown();
// CONFIG includes CONFIG2. CONFIG2 includes CONFIG_FOR_ENUM
out=new BufferedWriter(new FileWriter(CONFIG_FOR_ENUM));
startConfig();
appendProperty("e", "SecondLevelInclude");
appendProperty("f", "SecondLevelInclude");
endConfig();
out=new BufferedWriter(new FileWriter(CONFIG2));
startConfig();
startInclude(CONFIG_FOR_ENUM);
endInclude();
appendProperty("c","FirstLevelInclude");
appendProperty("d","FirstLevelInclude");
endConfig();
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
startInclude(CONFIG2);
endInclude();
appendProperty("a", "1");
appendProperty("b", "2");
endConfig();
// Add CONFIG as an InputStream resource.
File file = new File(CONFIG);
BufferedInputStream bis =
new BufferedInputStream(new FileInputStream(file));
conf.addResource(bis);
// The first time the conf is parsed, verify that all properties were read
// from all levels of includes.
assertEquals("1", conf.get("a"));
assertEquals("2", conf.get("b"));
assertEquals("FirstLevelInclude", conf.get("c"));
assertEquals("FirstLevelInclude", conf.get("d"));
assertEquals("SecondLevelInclude", conf.get("e"));
assertEquals("SecondLevelInclude", conf.get("f"));
// Add another resource to the conf.
out=new BufferedWriter(new FileWriter(CONFIG_MULTI_BYTE));
startConfig();
appendProperty("g", "3");
appendProperty("h", "4");
endConfig();
Path fileResource = new Path(CONFIG_MULTI_BYTE);
conf.addResource(fileResource);
// Verify that all properties were read from all levels of includes the
// second time the conf is parsed.
assertEquals("1", conf.get("a"));
assertEquals("2", conf.get("b"));
assertEquals("FirstLevelInclude", conf.get("c"));
assertEquals("FirstLevelInclude", conf.get("d"));
assertEquals("SecondLevelInclude", conf.get("e"));
assertEquals("SecondLevelInclude", conf.get("f"));
assertEquals("3", conf.get("g"));
assertEquals("4", conf.get("h"));
tearDown();
}
public void testOrderOfDuplicatePropertiesWithInclude() throws Exception {
tearDown();
// Property "a" is set to different values inside and outside of includes.
out=new BufferedWriter(new FileWriter(CONFIG2));
startConfig();
appendProperty("a", "a-InsideInclude");
appendProperty("b", "b-InsideInclude");
endConfig();
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("a","a-OutsideInclude");
startInclude(CONFIG2);
endInclude();
appendProperty("b","b-OutsideInclude");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals("a-InsideInclude", conf.get("a"));
assertEquals("b-OutsideInclude", conf.get("b"));
tearDown();
}
public void testRelativeIncludes() throws Exception {
tearDown();
String relConfig = new File("./tmp/test-config.xml").getAbsolutePath();