honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Decouple more classes from XContentBuilder and make builder strict (#29197) 

This commit decouples `BytesRef`, `Releaseable`, and `TimeValue` from
XContentBuilder, and paves the way for doupling `ByteSizeValue` as well. It
moves much of the Lucene and Joda encoding into a new SPI extension that is
loaded by XContentBuilder to know how to encode these values.

Part of doing this also allows us to make JSON encoding strict, as we no longer
allow just any old object to be passed (in the past it was possible to get json
that was `"field": "java.lang.Object@d8355a8"` if no one was careful about what
was passed in).

Relates to #28504
This commit is contained in:
Lee Hinman 2018-03-22 08:18:55 -06:00 committed by GitHub
parent 6c3278b8e8
commit 7d1de890b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 171 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
server/src/main/java/org/elasticsearch/Version.java
View File

@ -25,6 +25,8 @@ import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.monitor.jvm.JvmInfo;
import java.io.IOException;
@ -34,7 +36,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class Version implements Comparable<Version> {
public class Version implements Comparable<Version>, ToXContentFragment {
/*
* The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA
* values below 25 are for alpha builder (since 5.0), and above 25 and below 50 are beta builds, and below 99 are RC builds, with 99
@ -418,6 +420,11 @@ public class Version implements Comparable<Version> {
return Integer.compare(this.id, other.id);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.value(toString());
}
/*
* We need the declared versions when computing the minimum compatibility version. As computing the declared versions uses reflection it
* is not cheap. Since computing the minimum compatibility version can occur often, we use this holder to compute the declared versions

9
server/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
import org.elasticsearch.common.io.stream.BytesStream;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.ByteArrayOutputStream;
@ -37,7 +38,7 @@ import java.util.function.ToIntBiFunction;
/**
* A reference to bytes.
*/
public abstract class BytesReference implements Accountable, Comparable<BytesReference> {
public abstract class BytesReference implements Accountable, Comparable<BytesReference>, ToXContentFragment {
private Integer hash = null; // we cache the hash of this reference since it can be quite costly to re-calculated it
@ -334,4 +335,10 @@ public abstract class BytesReference implements Accountable, Comparable<BytesRef
return input.skip(n);
}
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
BytesRef bytes = toBytesRef();
return builder.value(bytes.bytes, bytes.offset, bytes.length);
}
}

7
server/src/main/java/org/elasticsearch/common/document/DocumentField.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.common.document;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -127,11 +128,7 @@ public class DocumentField implements Streamable, ToXContentFragment, Iterable<O
// Stored fields values are converted using MappedFieldType#valueForDisplay.
// As a result they can either be Strings, Numbers, or Booleans, that's
// all.
if (value instanceof BytesReference) {
builder.binaryValue(((BytesReference) value).toBytesRef());
} else {
builder.value(value);
}
builder.value(value);
}
builder.endArray();
return builder;

4
server/src/main/java/org/elasticsearch/common/text/Text.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.common.text;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.ToXContent;
@ -125,7 +126,8 @@ public final class Text implements Comparable<Text>, ToXContentFragment {
} else {
// TODO: TextBytesOptimization we can use a buffer here to convert it? maybe add a
// request to jackson to support InputStream as well?
return builder.utf8Value(this.bytes().toBytesRef());
BytesRef br = this.bytes().toBytesRef();
return builder.utf8Value(br.bytes, br.offset, br.length);
}
}
}

10
server/src/main/java/org/elasticsearch/common/transport/TransportAddress.java
View File

@ -23,6 +23,9 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.net.InetAddress;
@ -32,7 +35,7 @@ import java.net.UnknownHostException;
/**
* A transport address used for IP socket address (wraps {@link java.net.InetSocketAddress}).
*/
public final class TransportAddress implements Writeable {
public final class TransportAddress implements Writeable, ToXContentFragment {
/**
* A <a href="https://en.wikipedia.org/wiki/0.0.0.0">non-routeable v4 meta transport address</a> that can be used for
@ -128,4 +131,9 @@ public final class TransportAddress implements Writeable {
public String toString() {
return NetworkAddress.format(address);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.value(toString());
}
}

10
server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
View File

@ -27,12 +27,15 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Locale;
import java.util.Objects;
public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue> {
public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue>, ToXContentFragment {
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(ByteSizeValue.class));
private final long size;
@ -269,4 +272,9 @@ public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue> {
long otherValue = other.size * other.unit.toBytes(1);
return Long.compare(thisValue, otherValue);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.value(toString());
}
}

10
server/src/main/java/org/elasticsearch/common/unit/TimeValue.java
View File

@ -24,6 +24,9 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.joda.time.format.PeriodFormat;
@ -40,7 +43,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
public class TimeValue implements Writeable, Comparable<TimeValue> {
public class TimeValue implements Writeable, Comparable<TimeValue>, ToXContentFragment {
/** How many nano-seconds in one milli-second */
public static final long NSEC_PER_MSEC = TimeUnit.NANOSECONDS.convert(1, TimeUnit.MILLISECONDS);
@ -398,4 +401,9 @@ public class TimeValue implements Writeable, Comparable<TimeValue> {
double otherValue = ((double) timeValue.duration) * timeValue.timeUnit.toNanos(1);
return Double.compare(thisValue, otherValue);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.value(toString());
}
}

59
server/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
View File

@ -19,10 +19,7 @@
package org.elasticsearch.common.xcontent;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.CollectionUtils;
import org.joda.time.DateTimeZone;
import org.joda.time.ReadableInstant;
@ -30,11 +27,13 @@ import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Path;
import java.time.ZonedDateTime;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
@ -49,7 +48,7 @@ import java.util.Set;
/**
* A utility to build XContent (ie json).
*/
public final class XContentBuilder implements Releasable, Flushable {
public final class XContentBuilder implements Closeable, Flushable {
/**
* Create a new {@link XContentBuilder} using the given {@link XContent} content.
@ -91,7 +90,6 @@ public final class XContentBuilder implements Releasable, Flushable {
writers.put(Boolean.class, (b, v) -> b.value((Boolean) v));
writers.put(Byte.class, (b, v) -> b.value((Byte) v));
writers.put(byte[].class, (b, v) -> b.value((byte[]) v));
writers.put(BytesRef.class, (b, v) -> b.binaryValue((BytesRef) v));
writers.put(Date.class, (b, v) -> b.value((Date) v));
writers.put(Double.class, (b, v) -> b.value((Double) v));
writers.put(double[].class, (b, v) -> b.values((double[]) v));
@ -105,12 +103,12 @@ public final class XContentBuilder implements Releasable, Flushable {
writers.put(short[].class, (b, v) -> b.values((short[]) v));
writers.put(String.class, (b, v) -> b.value((String) v));
writers.put(String[].class, (b, v) -> b.values((String[]) v));
writers.put(Locale.class, (b, v) -> b.value(v.toString()));
writers.put(Class.class, (b, v) -> b.value(v.toString()));
writers.put(ZonedDateTime.class, (b, v) -> b.value(v.toString()));
Map<Class<?>, HumanReadableTransformer> humanReadableTransformer = new HashMap<>();
// These will be moved to a different class at a later time to decouple them from XContentBuilder
humanReadableTransformer.put(TimeValue.class, v -> ((TimeValue) v).millis());
humanReadableTransformer.put(ByteSizeValue.class, v -> ((ByteSizeValue) v).getBytes());
// Load pluggable extensions
for (XContentBuilderExtension service : ServiceLoader.load(XContentBuilderExtension.class)) {
@ -613,49 +611,25 @@ public final class XContentBuilder implements Releasable, Flushable {
}
/**
* Writes the binary content of the given {@link BytesRef}.
*
* Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
*/
public XContentBuilder field(String name, BytesRef value) throws IOException {
return field(name).binaryValue(value);
}
/**
* Writes the binary content of the given {@link BytesRef} as UTF-8 bytes.
* Writes the binary content of the given byte array as UTF-8 bytes.
*
* Use {@link XContentParser#charBuffer()} to read the value back
*/
public XContentBuilder utf8Field(String name, BytesRef value) throws IOException {
return field(name).utf8Value(value);
public XContentBuilder utf8Field(String name, byte[] bytes, int offset, int length) throws IOException {
return field(name).utf8Value(bytes, offset, length);
}
/**
* Writes the binary content of the given {@link BytesRef}.
*
* Use {@link org.elasticsearch.common.xcontent.XContentParser#binaryValue()} to read the value back
*/
public XContentBuilder binaryValue(BytesRef value) throws IOException {
if (value == null) {
return nullValue();
}
value(value.bytes, value.offset, value.length);
return this;
}
/**
* Writes the binary content of the given {@link BytesRef} as UTF-8 bytes.
* Writes the binary content of the given byte array as UTF-8 bytes.
*
* Use {@link XContentParser#charBuffer()} to read the value back
*/
public XContentBuilder utf8Value(BytesRef value) throws IOException {
if (value == null) {
return nullValue();
}
generator.writeUTF8String(value.bytes, value.offset, value.length);
public XContentBuilder utf8Value(byte[] bytes, int offset, int length) throws IOException {
generator.writeUTF8String(bytes, offset, length);
return this;
}
////////////////////////////////////////////////////////////////////////////
// Date
//////////////////////////////////
@ -793,10 +767,11 @@ public final class XContentBuilder implements Releasable, Flushable {
value((ReadableInstant) value);
} else if (value instanceof ToXContent) {
value((ToXContent) value);
} else {
// This is a "value" object (like enum, DistanceUnit, etc) just toString() it
// (yes, it can be misleading when toString a Java class, but really, jackson should be used in that case)
} else if (value instanceof Enum<?>) {
// Write out the Enum toString
value(Objects.toString(value));
} else {
throw new IllegalArgumentException("cannot write xcontent for unknown value of type " + value.getClass());
}
}

78
server/src/main/java/org/elasticsearch/common/xcontent/XContentElasticsearchExtension.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.xcontent;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.joda.time.DateTimeZone;
import org.joda.time.tz.CachedDateTimeZone;
import org.joda.time.tz.FixedDateTimeZone;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
/**
* SPI extensions for Elasticsearch-specific classes (like the Lucene or Joda
* dependency classes) that need to be encoded by {@link XContentBuilder} in a
* specific way.
*/
public class XContentElasticsearchExtension implements XContentBuilderExtension {
@Override
public Map<Class<?>, XContentBuilder.Writer> getXContentWriters() {
Map<Class<?>, XContentBuilder.Writer> writers = new HashMap<>();
// Fully-qualified here to reduce ambiguity around our (ES') Version class
writers.put(org.apache.lucene.util.Version.class, (b, v) -> b.value(Objects.toString(v)));
writers.put(DateTimeZone.class, (b, v) -> b.value(Objects.toString(v)));
writers.put(CachedDateTimeZone.class, (b, v) -> b.value(Objects.toString(v)));
writers.put(FixedDateTimeZone.class, (b, v) -> b.value(Objects.toString(v)));
writers.put(BytesReference.class, (b, v) -> {
if (v == null) {
b.nullValue();
} else {
BytesRef bytes = ((BytesReference) v).toBytesRef();
b.value(bytes.bytes, bytes.offset, bytes.length);
}
});
writers.put(BytesRef.class, (b, v) -> {
if (v == null) {
b.nullValue();
} else {
BytesRef bytes = (BytesRef) v;
b.value(bytes.bytes, bytes.offset, bytes.length);
}
});
return writers;
}
@Override
public Map<Class<?>, XContentBuilder.HumanReadableTransformer> getXContentHumanReadableTransformers() {
Map<Class<?>, XContentBuilder.HumanReadableTransformer> transformers = new HashMap<>();
transformers.put(TimeValue.class, v -> ((TimeValue) v).millis());
transformers.put(ByteSizeValue.class, v -> ((ByteSizeValue) v).getBytes());
return transformers;
}
}

4
server/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
View File

@ -228,7 +228,6 @@ public interface XContentParser extends Closeable {
* Reads a plain binary value that was written via one of the following methods:
*
* <ul>
* <li>{@link XContentBuilder#field(String, org.apache.lucene.util.BytesRef)}</li>
* <li>{@link XContentBuilder#field(String, byte[], int, int)}}</li>
* <li>{@link XContentBuilder#field(String, byte[])}}</li>
* </ul>
@ -236,8 +235,7 @@ public interface XContentParser extends Closeable {
* as well as via their <code>String</code> variants of the separated value methods.
* Note: Do not use this method to read values written with:
* <ul>
* <li>{@link XContentBuilder#utf8Field(String, org.apache.lucene.util.BytesRef)}</li>
* <li>{@link XContentBuilder#utf8Field(String, org.apache.lucene.util.BytesRef)}</li>
* <li>{@link XContentBuilder#utf8Field(String, byte[], int, int)}</li>
* </ul>
*
* these methods write UTF-8 encoded strings and must be read through:

10
server/src/main/java/org/elasticsearch/index/shard/ShardId.java
View File

@ -23,6 +23,9 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.Index;
import java.io.IOException;
@ -30,7 +33,7 @@ import java.io.IOException;
/**
* Allows for shard level components to be injected with the shard id.
*/
public class ShardId implements Streamable, Comparable<ShardId> {
public class ShardId implements Streamable, Comparable<ShardId>, ToXContentFragment {
private Index index;
@ -137,4 +140,9 @@ public class ShardId implements Streamable, Comparable<ShardId> {
}
return Integer.compare(shardId, o.getId());
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.value(toString());
}
}

3
server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
View File

@ -266,7 +266,8 @@ public class BlobStoreIndexShardSnapshot implements ToXContentFragment {
}
if (file.metadata.hash() != null && file.metadata().hash().length > 0) {
builder.field(META_HASH, file.metadata.hash());
BytesRef br = file.metadata.hash();
builder.field(META_HASH, br.bytes, br.offset, br.length);
}
builder.endObject();
}

2
server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
View File

@ -120,7 +120,7 @@ public class DateHistogramValuesSourceBuilder extends CompositeValuesSourceBuild
builder.field(Histogram.INTERVAL_FIELD.getPreferredName(), dateHistogramInterval.toString());
}
if (timeZone != null) {
builder.field("time_zone", timeZone);
builder.field("time_zone", timeZone.toString());
}
}

10
server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramInterval.java
View File

@ -22,6 +22,9 @@ package org.elasticsearch.search.aggregations.bucket.histogram;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Objects;
@ -29,7 +32,7 @@ import java.util.Objects;
/**
* The interval the date histogram is based on.
*/
public class DateHistogramInterval implements Writeable {
public class DateHistogramInterval implements Writeable, ToXContentFragment {
public static final DateHistogramInterval SECOND = new DateHistogramInterval("1s");
public static final DateHistogramInterval MINUTE = new DateHistogramInterval("1m");
@ -100,4 +103,9 @@ public class DateHistogramInterval implements Writeable {
DateHistogramInterval other = (DateHistogramInterval) obj;
return Objects.equals(expression, other.expression);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.value(toString());
}
}

2
server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
View File

@ -338,7 +338,7 @@ public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB
builder.field("format", format);
}
if (timeZone != null) {
builder.field("time_zone", timeZone);
builder.field("time_zone", timeZone.toString());
}
if (valueType != null) {
builder.field("value_type", valueType.getPreferredName());

1
server/src/main/resources/META-INF/services/org.elasticsearch.common.xcontent.XContentBuilderExtension Normal file
View File

@ -0,0 +1 @@
org.elasticsearch.common.xcontent.XContentElasticsearchExtension

6
server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
View File

@ -326,14 +326,14 @@ public abstract class BaseXContentTestCase extends ESTestCase {
}
public void testBinaryUTF8() throws Exception {
assertResult("{'utf8':null}", () -> builder().startObject().utf8Field("utf8", null).endObject());
assertResult("{'utf8':null}", () -> builder().startObject().nullField("utf8").endObject());
final BytesRef randomBytesRef = new BytesRef(randomBytes());
XContentBuilder builder = builder().startObject();
if (randomBoolean()) {
builder.utf8Field("utf8", randomBytesRef);
builder.utf8Field("utf8", randomBytesRef.bytes, randomBytesRef.offset, randomBytesRef.length);
} else {
builder.field("utf8").utf8Value(randomBytesRef);
builder.field("utf8").utf8Value(randomBytesRef.bytes, randomBytesRef.offset, randomBytesRef.length);
}
builder.endObject();

2
server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java
View File

@ -68,7 +68,7 @@ public class BinaryDVFieldDataTests extends AbstractFieldDataTestCase {
writer.addDocument(d.rootDoc());
BytesRef bytes1 = randomBytes();
doc = XContentFactory.jsonBuilder().startObject().field("field", bytes1).endObject();
doc = XContentFactory.jsonBuilder().startObject().field("field", bytes1.bytes, bytes1.offset, bytes1.length).endObject();
d = mapper.parse(SourceToParse.source("test", "test", "2", BytesReference.bytes(doc), XContentType.JSON));
writer.addDocument(d.rootDoc());