honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ingest: Don't allow circular referencing of named patterns in the grok processor. 

Otherwise the grok code throws a stackoverflow error.

Closes #29257
This commit is contained in:
Martijn van Groningen 2018-03-29 16:20:01 +02:00
parent 2c20f7a164
commit 9da95efa41
No known key found for this signature in database
GPG Key ID: AB236F4FCF2AF12A
2 changed files with 114 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
libs/grok/src/main/java/org/elasticsearch/grok/Grok.java
View File

@ -34,8 +34,10 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Collections;
@ -74,8 +76,6 @@ public final class Grok {
private final Map<String, String> patternBank;
private final boolean namedCaptures;
private final Regex compiledExpression;
private final String expression;
public Grok(Map<String, String> patternBank, String grokPattern) {
this(patternBank, grokPattern, true);
@ -86,11 +86,59 @@ public final class Grok {
this.patternBank = patternBank;
this.namedCaptures = namedCaptures;
this.expression = toRegex(grokPattern);
for (Map.Entry<String, String> entry : patternBank.entrySet()) {
String name = entry.getKey();
String pattern = entry.getValue();
forbidCircularReferences(name, new ArrayList<>(), pattern);
}
String expression = toRegex(grokPattern);
byte[] expressionBytes = expression.getBytes(StandardCharsets.UTF_8);
this.compiledExpression = new Regex(expressionBytes, 0, expressionBytes.length, Option.DEFAULT, UTF8Encoding.INSTANCE);
}
/**
* Checks whether patterns reference each other in a circular manner and if so fail with an exception
*
* In a pattern, anything between <code>%{</code> and <code>}</code> or <code>:</code> is considered
* a reference to another named pattern. This method will navigate to all these named patterns and
* check for a circular reference.
*/
private void forbidCircularReferences(String patternName, List<String> path, String pattern) {
if (pattern.contains("%{" + patternName + "}") || pattern.contains("%{" + patternName + ":")) {
String message;
if (path.isEmpty()) {
message = "circular reference in pattern [" + patternName + "][" + pattern + "]";
} else {
message = "circular reference in pattern [" + path.remove(path.size() - 1) + "][" + pattern +
"] back to pattern [" + patternName + "]";
// add rest of the path:
if (path.isEmpty() == false) {
message += " via patterns [" + String.join("=>", path) + "]";
}
}
throw new IllegalArgumentException(message);
}
for (int i = pattern.indexOf("%{"); i != -1; i = pattern.indexOf("%{", i + 1)) {
int begin = i + 2;
int brackedIndex = pattern.indexOf('}', begin);
int columnIndex = pattern.indexOf(':', begin);
int end;
if (brackedIndex != -1 && columnIndex == -1) {
end = brackedIndex;
} else if (columnIndex != -1 && brackedIndex == -1) {
end = columnIndex;
} else if (brackedIndex != -1 && columnIndex != -1) {
end = Math.min(brackedIndex, columnIndex);
} else {
throw new IllegalArgumentException("pattern [" + pattern + "] has circular references to other pattern definitions");
}
String otherPatternName = pattern.substring(begin, end);
path.add(otherPatternName);
forbidCircularReferences(patternName, path, patternBank.get(otherPatternName));
}
}
public String groupMatch(String name, Region region, String pattern) {
try {
@ -125,10 +173,12 @@ public final class Grok {
String patternName = groupMatch(PATTERN_GROUP, region, grokPattern);
String pattern = patternBank.get(patternName);
if (pattern == null) {
throw new IllegalArgumentException("Unable to find pattern [" + patternName + "] in Grok's pattern dictionary");
}
if (pattern.contains("%{" + patternName + "}") || pattern.contains("%{" + patternName + ":")) {
throw new IllegalArgumentException("circular reference in pattern back [" + patternName + "]");
}
String grokPart;
if (namedCaptures && subName != null) {

60
libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java
View File

@ -28,6 +28,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
@ -205,6 +206,65 @@ public class GrokTests extends ESTestCase {
assertEquals(expected, actual);
}
public void testCircularReference() {
Exception e = expectThrows(IllegalArgumentException.class, () -> {
Map<String, String> bank = new HashMap<>();
bank.put("NAME", "!!!%{NAME}!!!");
String pattern = "%{NAME}";
new Grok(bank, pattern, false);
});
assertEquals("circular reference in pattern [NAME][!!!%{NAME}!!!]", e.getMessage());
e = expectThrows(IllegalArgumentException.class, () -> {
Map<String, String> bank = new HashMap<>();
bank.put("NAME", "!!!%{NAME:name}!!!");
String pattern = "%{NAME}";
new Grok(bank, pattern, false);
});
assertEquals("circular reference in pattern [NAME][!!!%{NAME:name}!!!]", e.getMessage());
e = expectThrows(IllegalArgumentException.class, () -> {
Map<String, String> bank = new HashMap<>();
bank.put("NAME", "!!!%{NAME:name:int}!!!");
String pattern = "%{NAME}";
new Grok(bank, pattern, false);
});
assertEquals("circular reference in pattern [NAME][!!!%{NAME:name:int}!!!]", e.getMessage());
e = expectThrows(IllegalArgumentException.class, () -> {
Map<String, String> bank = new TreeMap<>();
bank.put("NAME1", "!!!%{NAME2}!!!");
bank.put("NAME2", "!!!%{NAME1}!!!");
String pattern = "%{NAME1}";
new Grok(bank, pattern, false);
});
assertEquals("circular reference in pattern [NAME2][!!!%{NAME1}!!!] back to pattern [NAME1]", e.getMessage());
e = expectThrows(IllegalArgumentException.class, () -> {
Map<String, String> bank = new TreeMap<>();
bank.put("NAME1", "!!!%{NAME2}!!!");
bank.put("NAME2", "!!!%{NAME3}!!!");
bank.put("NAME3", "!!!%{NAME1}!!!");
String pattern = "%{NAME1}";
new Grok(bank, pattern, false);
});
assertEquals("circular reference in pattern [NAME3][!!!%{NAME1}!!!] back to pattern [NAME1] via patterns [NAME2]",
e.getMessage());
e = expectThrows(IllegalArgumentException.class, () -> {
Map<String, String> bank = new TreeMap<>();
bank.put("NAME1", "!!!%{NAME2}!!!");
bank.put("NAME2", "!!!%{NAME3}!!!");
bank.put("NAME3", "!!!%{NAME4}!!!");
bank.put("NAME4", "!!!%{NAME5}!!!");
bank.put("NAME5", "!!!%{NAME1}!!!");
String pattern = "%{NAME1}";
new Grok(bank, pattern, false);
});
assertEquals("circular reference in pattern [NAME5][!!!%{NAME1}!!!] back to pattern [NAME1] " +
"via patterns [NAME2=>NAME3=>NAME4]", e.getMessage());
}
public void testBooleanCaptures() {
Map<String, String> bank = new HashMap<>();