honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Trim the JSON source in indexing slow logs (#38081) 

The '{' as a first character in log line is causing problems for beats when parsing plaintext logs. This can happen if the submitted document has an additional '\n' at the beginning and we are not reformatting. 
Trimming the source part of a SlogLog solves that and keeps the logs readable.

closes #38080
This commit is contained in:
Przemyslaw Gomulka 2019-02-01 08:12:12 +01:00 committed by GitHub
parent 6fcbd07420
commit 2758578570
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 18 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
View File

@ -194,7 +194,7 @@ public final class IndexingSlowLog implements IndexingOperationListener {
}
try {
String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType());
sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog)).append("]");
sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog).trim()).append("]");
} catch (IOException e) {
sb.append(", source[_failed_to_convert_[").append(e.getMessage()).append("]]");
/*

17
server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
View File

@ -38,6 +38,7 @@ import java.io.IOException;
import java.io.UncheckedIOException;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasToString;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.not;
@ -127,6 +128,22 @@ public class IndexingSlowLogTests extends ESTestCase {
assertTrue(log.isReformat());
}
public void testReformatIsFalseAndSourceIsTrim() {
String json = "\n\n{ \"fieldName\": 123 } \n ";
BytesReference source = new BytesArray(json);
ParsedDocument pd = new ParsedDocument(new NumericDocValuesField("version", 1),
SeqNoFieldMapper.SequenceIDFields.emptySeqID(), "id",
"test", null, null, source, XContentType.JSON, null);
Index index = new Index("foo", "123");
// Turning off reformatting so the document is in logs as provided
SlowLogParsedDocumentPrinter p = new SlowLogParsedDocumentPrinter(index, pd, 10, false, 1000);
String logLine = p.toString();
//expect the new lines and white characters to be trimmed
assertThat(logLine, containsString("source[{"));
assertThat(logLine.split("\n").length, equalTo(1));
}
public void testLevelSetting() {
SlowLogLevel level = randomFrom(SlowLogLevel.values());
IndexMetaData metaData = newIndexMeta("index", Settings.builder()