From f51da9c4d1423c2ac92eb4f40e973264e7e968cc Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Mon, 2 Jul 2018 18:31:21 +0200 Subject: [PATCH] HADOOP-15554. Improve JIT performance for Configuration parsing. Contributed by Todd Lipcon. --- .../org/apache/hadoop/conf/Configuration.java | 458 +++++++++++------- 1 file changed, 276 insertions(+), 182 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index b1125e588c0..a78e3119993 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -41,6 +41,7 @@ import java.io.Writer; import java.lang.ref.WeakReference; import java.net.InetSocketAddress; import java.net.JarURLConnection; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; @@ -2981,187 +2982,11 @@ public class Configuration implements Iterable>, if(returnCachedProperties) { toAddTo = new Properties(); } - DeprecationContext deprecations = deprecationContext.get(); - StringBuilder token = new StringBuilder(); - String confName = null; - String confValue = null; - String confInclude = null; - String confTag = null; - boolean confFinal = false; - boolean fallbackAllowed = false; - boolean fallbackEntered = false; - boolean parseToken = false; - LinkedList confSource = new LinkedList(); - - while (reader.hasNext()) { - switch (reader.next()) { - case XMLStreamConstants.START_ELEMENT: - switch (reader.getLocalName()) { - case "property": - confName = null; - confValue = null; - confFinal = false; - confTag = null; - confSource.clear(); - - // First test for short format configuration - int attrCount = reader.getAttributeCount(); - for (int i = 0; i < attrCount; i++) { - String propertyAttr = reader.getAttributeLocalName(i); - if ("name".equals(propertyAttr)) { - confName = StringInterner.weakIntern( - reader.getAttributeValue(i)); - } else if ("value".equals(propertyAttr)) { - confValue = StringInterner.weakIntern( - reader.getAttributeValue(i)); - } else if ("final".equals(propertyAttr)) { - confFinal = "true".equals(reader.getAttributeValue(i)); - } else if ("source".equals(propertyAttr)) { - confSource.add(StringInterner.weakIntern( - reader.getAttributeValue(i))); - } else if ("tag".equals(propertyAttr)) { - confTag = StringInterner - .weakIntern(reader.getAttributeValue(i)); - } - } - break; - case "name": - case "value": - case "final": - case "source": - case "tag": - parseToken = true; - token.setLength(0); - break; - case "include": - // Determine href for xi:include - confInclude = null; - attrCount = reader.getAttributeCount(); - for (int i = 0; i < attrCount; i++) { - String attrName = reader.getAttributeLocalName(i); - if ("href".equals(attrName)) { - confInclude = reader.getAttributeValue(i); - } - } - if (confInclude == null) { - break; - } - if (isRestricted) { - throw new RuntimeException("Error parsing resource " + wrapper - + ": XInclude is not supported for restricted resources"); - } - // Determine if the included resource is a classpath resource - // otherwise fallback to a file resource - // xi:include are treated as inline and retain current source - URL include = getResource(confInclude); - if (include != null) { - Resource classpathResource = new Resource(include, name, - wrapper.isParserRestricted()); - loadResource(properties, classpathResource, quiet); - } else { - URL url; - try { - url = new URL(confInclude); - url.openConnection().connect(); - } catch (IOException ioe) { - File href = new File(confInclude); - if (!href.isAbsolute()) { - // Included resources are relative to the current resource - File baseFile = new File(name).getParentFile(); - href = new File(baseFile, href.getPath()); - } - if (!href.exists()) { - // Resource errors are non-fatal iff there is 1 xi:fallback - fallbackAllowed = true; - break; - } - url = href.toURI().toURL(); - } - Resource uriResource = new Resource(url, name, - wrapper.isParserRestricted()); - loadResource(properties, uriResource, quiet); - } - break; - case "fallback": - fallbackEntered = true; - break; - case "configuration": - break; - default: - break; - } - break; - - case XMLStreamConstants.CHARACTERS: - if (parseToken) { - char[] text = reader.getTextCharacters(); - token.append(text, reader.getTextStart(), reader.getTextLength()); - } - break; - - case XMLStreamConstants.END_ELEMENT: - switch (reader.getLocalName()) { - case "name": - if (token.length() > 0) { - confName = StringInterner.weakIntern(token.toString().trim()); - } - break; - case "value": - if (token.length() > 0) { - confValue = StringInterner.weakIntern(token.toString()); - } - break; - case "final": - confFinal = "true".equals(token.toString()); - break; - case "source": - confSource.add(StringInterner.weakIntern(token.toString())); - break; - case "tag": - if (token.length() > 0) { - confTag = StringInterner.weakIntern(token.toString()); - } - break; - case "include": - if (fallbackAllowed && !fallbackEntered) { - throw new IOException("Fetch fail on include for '" - + confInclude + "' with no fallback while loading '" - + name + "'"); - } - fallbackAllowed = false; - fallbackEntered = false; - break; - case "property": - if (confName == null || (!fallbackAllowed && fallbackEntered)) { - break; - } - confSource.add(name); - // Read tags and put them in propertyTagsMap - if (confTag != null) { - readTagFromConfig(confTag, confName, confValue, confSource); - } - - DeprecatedKeyInfo keyInfo = - deprecations.getDeprecatedKeyMap().get(confName); - if (keyInfo != null) { - keyInfo.clearAccessed(); - for (String key : keyInfo.newKeys) { - // update new keys with deprecated key's value - loadProperty(toAddTo, name, key, confValue, confFinal, - confSource.toArray(new String[confSource.size()])); - } - } else { - loadProperty(toAddTo, name, confName, confValue, confFinal, - confSource.toArray(new String[confSource.size()])); - } - break; - default: - break; - } - default: - break; - } + List items = new Parser(reader, wrapper, quiet).parse(); + for (ParsedItem item : items) { + loadProperty(toAddTo, item.name, item.key, item.value, + item.isFinal, item.sources); } reader.close(); @@ -3179,6 +3004,275 @@ public class Configuration implements Iterable>, } } + private static class ParsedItem { + String name; + String key; + String value; + boolean isFinal; + String[] sources; + + ParsedItem(String name, String key, String value, + boolean isFinal, String[] sources) { + this.name = name; + this.key = key; + this.value = value; + this.isFinal = isFinal; + this.sources = sources; + } + } + + /** + * Parser to consume SAX stream of XML elements from a Configuration. + */ + private class Parser { + private final XMLStreamReader2 reader; + private final Resource wrapper; + private final String name; + private final String[] nameSingletonArray; + private final boolean isRestricted; + private final boolean quiet; + + DeprecationContext deprecations = deprecationContext.get(); + + private StringBuilder token = new StringBuilder(); + private String confName = null; + private String confValue = null; + private String confInclude = null; + private String confTag = null; + private boolean confFinal = false; + private boolean fallbackAllowed = false; + private boolean fallbackEntered = false; + private boolean parseToken = false; + private List confSource = new ArrayList<>(); + private List results = new ArrayList<>(); + + Parser(XMLStreamReader2 reader, + Resource wrapper, + boolean quiet) { + this.reader = reader; + this.wrapper = wrapper; + this.name = wrapper.getName(); + this.nameSingletonArray = new String[]{ name }; + this.isRestricted = wrapper.isParserRestricted(); + this.quiet = quiet; + + } + + List parse() throws IOException, XMLStreamException { + while (reader.hasNext()) { + parseNext(); + } + return results; + } + + private void handleStartElement() throws MalformedURLException { + switch (reader.getLocalName()) { + case "property": + handleStartProperty(); + break; + + case "name": + case "value": + case "final": + case "source": + case "tag": + parseToken = true; + token.setLength(0); + break; + case "include": + handleInclude(); + break; + case "fallback": + fallbackEntered = true; + break; + case "configuration": + break; + default: + break; + } + } + + private void handleStartProperty() { + confName = null; + confValue = null; + confFinal = false; + confTag = null; + confSource.clear(); + + // First test for short format configuration + int attrCount = reader.getAttributeCount(); + for (int i = 0; i < attrCount; i++) { + String propertyAttr = reader.getAttributeLocalName(i); + if ("name".equals(propertyAttr)) { + confName = StringInterner.weakIntern( + reader.getAttributeValue(i)); + } else if ("value".equals(propertyAttr)) { + confValue = StringInterner.weakIntern( + reader.getAttributeValue(i)); + } else if ("final".equals(propertyAttr)) { + confFinal = "true".equals(reader.getAttributeValue(i)); + } else if ("source".equals(propertyAttr)) { + confSource.add(StringInterner.weakIntern( + reader.getAttributeValue(i))); + } else if ("tag".equals(propertyAttr)) { + confTag = StringInterner + .weakIntern(reader.getAttributeValue(i)); + } + } + } + + private void handleInclude() throws MalformedURLException { + // Determine href for xi:include + confInclude = null; + int attrCount = reader.getAttributeCount(); + for (int i = 0; i < attrCount; i++) { + String attrName = reader.getAttributeLocalName(i); + if ("href".equals(attrName)) { + confInclude = reader.getAttributeValue(i); + } + } + if (confInclude == null) { + return; + } + if (isRestricted) { + throw new RuntimeException("Error parsing resource " + wrapper + + ": XInclude is not supported for restricted resources"); + } + // Determine if the included resource is a classpath resource + // otherwise fallback to a file resource + // xi:include are treated as inline and retain current source + URL include = getResource(confInclude); + if (include != null) { + Resource classpathResource = new Resource(include, name, + wrapper.isParserRestricted()); + // This is only called recursively while the lock is already held + // by this thread, but synchronizing avoids a findbugs warning. + synchronized (Configuration.this) { + loadResource(properties, classpathResource, quiet); + } + } else { + URL url; + try { + url = new URL(confInclude); + url.openConnection().connect(); + } catch (IOException ioe) { + File href = new File(confInclude); + if (!href.isAbsolute()) { + // Included resources are relative to the current resource + File baseFile = new File(name).getParentFile(); + href = new File(baseFile, href.getPath()); + } + if (!href.exists()) { + // Resource errors are non-fatal iff there is 1 xi:fallback + fallbackAllowed = true; + return; + } + url = href.toURI().toURL(); + } + Resource uriResource = new Resource(url, name, + wrapper.isParserRestricted()); + // This is only called recursively while the lock is already held + // by this thread, but synchronizing avoids a findbugs warning. + synchronized (Configuration.this) { + loadResource(properties, uriResource, quiet); + } + } + } + + void handleEndElement() throws IOException { + String tokenStr = token.toString(); + switch (reader.getLocalName()) { + case "name": + if (token.length() > 0) { + confName = StringInterner.weakIntern(tokenStr.trim()); + } + break; + case "value": + if (token.length() > 0) { + confValue = StringInterner.weakIntern(tokenStr); + } + break; + case "final": + confFinal = "true".equals(tokenStr); + break; + case "source": + confSource.add(StringInterner.weakIntern(tokenStr)); + break; + case "tag": + if (token.length() > 0) { + confTag = StringInterner.weakIntern(tokenStr); + } + break; + case "include": + if (fallbackAllowed && !fallbackEntered) { + throw new IOException("Fetch fail on include for '" + + confInclude + "' with no fallback while loading '" + + name + "'"); + } + fallbackAllowed = false; + fallbackEntered = false; + break; + case "property": + handleEndProperty(); + break; + default: + break; + } + } + + void handleEndProperty() { + if (confName == null || (!fallbackAllowed && fallbackEntered)) { + return; + } + String[] confSourceArray; + if (confSource.isEmpty()) { + confSourceArray = nameSingletonArray; + } else { + confSource.add(name); + confSourceArray = confSource.toArray(new String[confSource.size()]); + } + + // Read tags and put them in propertyTagsMap + if (confTag != null) { + readTagFromConfig(confTag, confName, confValue, confSourceArray); + } + + DeprecatedKeyInfo keyInfo = + deprecations.getDeprecatedKeyMap().get(confName); + + if (keyInfo != null) { + keyInfo.clearAccessed(); + for (String key : keyInfo.newKeys) { + // update new keys with deprecated key's value + results.add(new ParsedItem( + name, key, confValue, confFinal, confSourceArray)); + } + } else { + results.add(new ParsedItem(name, confName, confValue, confFinal, + confSourceArray)); + } + } + + void parseNext() throws IOException, XMLStreamException { + switch (reader.next()) { + case XMLStreamConstants.START_ELEMENT: + handleStartElement(); + break; + case XMLStreamConstants.CHARACTERS: + if (parseToken) { + char[] text = reader.getTextCharacters(); + token.append(text, reader.getTextStart(), reader.getTextLength()); + } + break; + case XMLStreamConstants.END_ELEMENT: + handleEndElement(); + break; + default: + break; + } + } + } + /** * Add tags defined in HADOOP_TAGS_SYSTEM, HADOOP_TAGS_CUSTOM. * @param prop @@ -3225,7 +3319,7 @@ public class Configuration implements Iterable>, * @param confSource */ private void readTagFromConfig(String attributeValue, String confName, String - confValue, List confSource) { + confValue, String[] confSource) { for (String tagStr : attributeValue.split(",")) { try { tagStr = tagStr.trim(); @@ -3243,7 +3337,7 @@ public class Configuration implements Iterable>, } catch (Exception ex) { // Log the exception at trace level. LOG.trace("Tag '{}' for property:{} Source:{}", tagStr, confName, - Arrays.toString(confSource.toArray()), ex); + confSource, ex); } } }