honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve resiliency to formatting JSON in server (#48706) 

Backport of #48553. Make a number of changes so that JSON in the server
directory is more resilient to automatic formatting. This covers:

   * Reformatting multiline JSON to embed whitespace in the strings
   * Add helper method `stripWhitespace()`, to strip whitespace from a JSON
     document using XContent methods. This is sometimes necessary where
     a test is comparing some machine-generated JSON with an expected
     value.
This commit is contained in:
Rory Hunter 2019-10-31 10:48:55 +00:00 committed by GitHub
parent eefa84bc94
commit d96976e2b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 199 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.common.xcontent;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.Compressor;
@ -161,6 +162,19 @@ public class XContentHelper {
return convertToJson(bytes, reformatJson, false, xContentType);
}
/**
* Accepts a JSON string, parses it and prints it without pretty-printing it. This is useful
* where a piece of JSON is formatted for legibility, but needs to be stripped of unnecessary
* whitespace e.g. for comparison in a test.
*
* @param json the JSON to format
* @return reformatted JSON
* @throws IOException if the reformatting fails, e.g. because the JSON is not well-formed
*/
public static String stripWhitespace(String json) throws IOException {
return convertToJson(new BytesArray(json), true, XContentType.JSON);
}
public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint, XContentType xContentType)
throws IOException {
Objects.requireNonNull(xContentType);

65
server/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContent;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.shard.IndexShardClosedException;
@ -56,36 +57,40 @@ public class SearchPhaseExecutionExceptionTests extends ESTestCase {
});
// Failures are grouped (by default)
assertEquals("{" +
"\"type\":\"search_phase_execution_exception\"," +
"\"reason\":\"all shards failed\"," +
"\"phase\":\"test\"," +
"\"grouped\":true," +
"\"failed_shards\":[" +
"{" +
"\"shard\":0," +
"\"index\":\"foo\"," +
"\"node\":\"node_1\"," +
"\"reason\":{" +
"\"type\":\"parsing_exception\"," +
"\"reason\":\"foobar\"," +
"\"line\":1," +
"\"col\":2" +
"}" +
"}," +
"{" +
"\"shard\":1," +
"\"index\":\"foo\"," +
"\"node\":\"node_2\"," +
"\"reason\":{" +
"\"type\":\"index_shard_closed_exception\"," +
"\"reason\":\"CurrentState[CLOSED] Closed\"," +
"\"index_uuid\":\"_na_\"," +
"\"shard\":\"1\"," +
"\"index\":\"foo\"" +
"}" +
"}" +
"]}", Strings.toString(exception));
final String expectedJson = XContentHelper.stripWhitespace(
"{"
+ " \"type\": \"search_phase_execution_exception\","
+ " \"reason\": \"all shards failed\","
+ " \"phase\": \"test\","
+ " \"grouped\": true,"
+ " \"failed_shards\": ["
+ " {"
+ " \"shard\": 0,"
+ " \"index\": \"foo\","
+ " \"node\": \"node_1\","
+ " \"reason\": {"
+ " \"type\": \"parsing_exception\","
+ " \"reason\": \"foobar\","
+ " \"line\": 1,"
+ " \"col\": 2"
+ " }"
+ " },"
+ " {"
+ " \"shard\": 1,"
+ " \"index\": \"foo\","
+ " \"node\": \"node_2\","
+ " \"reason\": {"
+ " \"type\": \"index_shard_closed_exception\","
+ " \"reason\": \"CurrentState[CLOSED] Closed\","
+ " \"index_uuid\": \"_na_\","
+ " \"shard\": \"1\","
+ " \"index\": \"foo\""
+ " }"
+ " }"
+ " ]"
+ "}"
);
assertEquals(expectedJson, Strings.toString(exception));
}
public void testToAndFromXContent() throws IOException {

25
server/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
View File

@ -165,18 +165,19 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
}
public void testParseJson() throws IOException {
String scriptSort = "{\n" +
"\"_script\" : {\n" +
"\"type\" : \"number\",\n" +
"\"script\" : {\n" +
"\"source\": \"doc['field_name'].value * factor\",\n" +
"\"params\" : {\n" +
"\"factor\" : 1.1\n" +
"}\n" +
"},\n" +
"\"mode\" : \"max\",\n" +
"\"order\" : \"asc\"\n" +
"} }\n";
String scriptSort = "{"
+ " \"_script\": {"
+ " \"type\": \"number\","
+ " \"script\": {"
+ " \"source\": \"doc['field_name'].value * factor\","
+ " \"params\": {"
+ " \"factor\": 1.1"
+ " }"
+ " },"
+ " \"mode\": \"max\","
+ " \"order\": \"asc\""
+ " }"
+ "}";
try (XContentParser parser = createParser(JsonXContent.jsonXContent, scriptSort)) {
parser.nextToken();
parser.nextToken();

36
server/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
View File

@ -52,6 +52,7 @@ import java.util.Collections;
import java.util.List;
import static java.util.Collections.emptyList;
import static org.elasticsearch.common.xcontent.XContentHelper.stripWhitespace;
import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName;
@ -131,19 +132,28 @@ public class SuggestTests extends ESTestCase {
Suggest suggest = new Suggest(Collections.singletonList(suggestion));
BytesReference xContent = toXContent(suggest, XContentType.JSON, randomBoolean());
assertEquals(
"{\"suggest\":"
+ "{\"suggestionName\":"
+ "[{\"text\":\"entryText\","
+ "\"offset\":42,"
+ "\"length\":313,"
+ "\"options\":[{\"text\":\"someText\","
+ "\"highlighted\":\"somethingHighlighted\","
+ "\"score\":1.3,"
+ "\"collate_match\":true}]"
+ "}]"
+ "}"
+"}",
xContent.utf8ToString());
stripWhitespace(
"{"
+ " \"suggest\": {"
+ " \"suggestionName\": ["
+ " {"
+ " \"text\": \"entryText\","
+ " \"offset\": 42,"
+ " \"length\": 313,"
+ " \"options\": ["
+ " {"
+ " \"text\": \"someText\","
+ " \"highlighted\": \"somethingHighlighted\","
+ " \"score\": 1.3,"
+ " \"collate_match\": true"
+ " }"
+ " ]"
+ " }"
+ " ]"
+ " }"
+ "}"
),
xContent.utf8ToString());
}
public void testFilter() throws Exception {

15
server/src/test/java/org/elasticsearch/search/suggest/SuggestionOptionTests.java
View File

@ -81,11 +81,14 @@ public class SuggestionOptionTests extends ESTestCase {
public void testToXContent() throws IOException {
Option option = new PhraseSuggestion.Entry.Option(new Text("someText"), new Text("somethingHighlighted"), 1.3f, true);
BytesReference xContent = toXContent(option, XContentType.JSON, randomBoolean());
assertEquals("{\"text\":\"someText\","
+ "\"highlighted\":\"somethingHighlighted\","
+ "\"score\":1.3,"
+ "\"collate_match\":true"
+ "}"
, xContent.utf8ToString());
assertEquals(
("{"
+ " \"text\": \"someText\","
+ " \"highlighted\": \"somethingHighlighted\","
+ " \"score\": 1.3,"
+ " \"collate_match\": true"
+ "}").replaceAll("\\s+", ""),
xContent.utf8ToString()
);
}
}

150
server/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java
View File

@ -164,17 +164,23 @@ public class SuggestionTests extends ESTestCase {
public void testUnknownSuggestionTypeThrows() throws IOException {
XContent xContent = JsonXContent.jsonXContent;
String suggestionString =
"{\"unknownType#suggestionName\":"
+ "[{\"text\":\"entryText\","
+ "\"offset\":42,"
+ "\"length\":313,"
+ "\"options\":[{\"text\":\"someText\","
+ "\"highlighted\":\"somethingHighlighted\","
+ "\"score\":1.3,"
+ "\"collate_match\":true}]"
+ "}]"
+ "}";
String suggestionString = ("{"
+ " \"unknownType#suggestionName\": ["
+ " {"
+ " \"text\": \"entryText\","
+ " \"offset\": 42,"
+ " \"length\": 313,"
+ " \"options\": ["
+ " {"
+ " \"text\": \"someText\","
+ " \"highlighted\": \"somethingHighlighted\","
+ " \"score\": 1.3,"
+ " \"collate_match\": true"
+ " }"
+ " ]"
+ " }"
+ " ]"
+ "}").replaceAll("\\s+", "");
try (XContentParser parser = xContent.createParser(xContentRegistry(),
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, suggestionString)) {
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
@ -195,18 +201,25 @@ public class SuggestionTests extends ESTestCase {
PhraseSuggestion suggestion = new PhraseSuggestion("suggestionName", 5);
suggestion.addTerm(entry);
BytesReference xContent = toXContent(suggestion, XContentType.JSON, params, randomBoolean());
assertEquals(
"{\"phrase#suggestionName\":[{"
+ "\"text\":\"entryText\","
+ "\"offset\":42,"
+ "\"length\":313,"
+ "\"options\":[{"
+ "\"text\":\"someText\","
+ "\"highlighted\":\"somethingHighlighted\","
+ "\"score\":1.3,"
+ "\"collate_match\":true}]"
+ "}]"
+ "}", xContent.utf8ToString());
assertEquals(("{"
+ " \"phrase#suggestionName\": ["
+ " {"
+ " \"text\": \"entryText\","
+ " \"offset\": 42,"
+ " \"length\": 313,"
+ " \"options\": ["
+ " {"
+ " \"text\": \"someText\","
+ " \"highlighted\": \"somethingHighlighted\","
+ " \"score\": 1.3,"
+ " \"collate_match\": true"
+ " }"
+ " ]"
+ " }"
+ " ]"
+ "}").replaceAll("\\s+", ""),
xContent.utf8ToString()
);
}
{
PhraseSuggestion.Entry.Option option = new PhraseSuggestion.Entry.Option(new Text("someText"), new Text("somethingHighlighted"),
@ -216,18 +229,25 @@ public class SuggestionTests extends ESTestCase {
PhraseSuggestion suggestion = new PhraseSuggestion("suggestionName", 5);
suggestion.addTerm(entry);
BytesReference xContent = toXContent(suggestion, XContentType.JSON, params, randomBoolean());
assertEquals(
"{\"phrase#suggestionName\":[{"
+ "\"text\":\"entryText\","
+ "\"offset\":42,"
+ "\"length\":313,"
+ "\"options\":[{"
+ "\"text\":\"someText\","
+ "\"highlighted\":\"somethingHighlighted\","
+ "\"score\":1.3,"
+ "\"collate_match\":true}]"
+ "}]"
+ "}", xContent.utf8ToString());
assertEquals(("{"
+ " \"phrase#suggestionName\": ["
+ " {"
+ " \"text\": \"entryText\","
+ " \"offset\": 42,"
+ " \"length\": 313,"
+ " \"options\": ["
+ " {"
+ " \"text\": \"someText\","
+ " \"highlighted\": \"somethingHighlighted\","
+ " \"score\": 1.3,"
+ " \"collate_match\": true"
+ " }"
+ " ]"
+ " }"
+ " ]"
+ "}").replaceAll("\\s+", ""),
xContent.utf8ToString()
);
}
{
TermSuggestion.Entry.Option option = new TermSuggestion.Entry.Option(new Text("someText"), 10, 1.3f);
@ -237,16 +257,24 @@ public class SuggestionTests extends ESTestCase {
suggestion.addTerm(entry);
BytesReference xContent = toXContent(suggestion, XContentType.JSON, params, randomBoolean());
assertEquals(
"{\"term#suggestionName\":[{"
+ "\"text\":\"entryText\","
+ "\"offset\":42,"
+ "\"length\":313,"
+ "\"options\":[{"
+ "\"text\":\"someText\","
+ "\"score\":1.3,"
+ "\"freq\":10}]"
+ "}]"
+ "}", xContent.utf8ToString());
("{"
+ " \"term#suggestionName\": ["
+ " {"
+ " \"text\": \"entryText\","
+ " \"offset\": 42,"
+ " \"length\": 313,"
+ " \"options\": ["
+ " {"
+ " \"text\": \"someText\","
+ " \"score\": 1.3,"
+ " \"freq\": 10"
+ " }"
+ " ]"
+ " }"
+ " ]"
+ "}").replaceAll("\\s+", ""),
xContent.utf8ToString()
);
}
{
Map<String, Set<String>> contexts = Collections.singletonMap("key", Collections.singleton("value"));
@ -257,16 +285,28 @@ public class SuggestionTests extends ESTestCase {
suggestion.addTerm(entry);
BytesReference xContent = toXContent(suggestion, XContentType.JSON, params, randomBoolean());
assertEquals(
"{\"completion#suggestionName\":[{"
+ "\"text\":\"entryText\","
+ "\"offset\":42,"
+ "\"length\":313,"
+ "\"options\":[{"
+ "\"text\":\"someText\","
+ "\"score\":1.3,"
+ "\"contexts\":{\"key\":[\"value\"]}"
+ "}]"
+ "}]}", xContent.utf8ToString());
("{"
+ " \"completion#suggestionName\": ["
+ " {"
+ " \"text\": \"entryText\","
+ " \"offset\": 42,"
+ " \"length\": 313,"
+ " \"options\": ["
+ " {"
+ " \"text\": \"someText\","
+ " \"score\": 1.3,"
+ " \"contexts\": {"
+ " \"key\": ["
+ " \"value\""
+ " ]"
+ " }"
+ " }"
+ " ]"
+ " }"
+ " ]"
+ "}").replaceAll("\\s+", ""),
xContent.utf8ToString()
);
}
}
}

18
server/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
View File

@ -210,14 +210,16 @@ public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCas
public void testMalformedJson() {
final String field = RandomStrings.randomAsciiOfLength(random(), 10).toLowerCase(Locale.ROOT);
String suggest = "{\n" +
" \"bad-payload\" : {\n" +
" \"text\" : \"the amsterdma meetpu\",\n" +
" \"term\" : {\n" +
" \"field\" : { \"" + field + "\" : \"bad-object\" }\n" +
" }\n" +
" }\n" +
"}";
String suggest = "{\n"
+ " \"bad-payload\" : {\n"
+ " \"text\" : \"the amsterdma meetpu\",\n"
+ " \"term\" : {\n"
+ " \"field\" : { \""
+ field
+ "\" : \"bad-object\" }\n"
+ " }\n"
+ " }\n"
+ "}";
try (XContentParser parser = createParser(JsonXContent.jsonXContent, suggest)) {
final SuggestBuilder suggestBuilder = SuggestBuilder.fromXContent(parser);
fail("Should not have been able to create SuggestBuilder from malformed JSON: " + suggestBuilder);