mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-17 02:14:54 +00:00
Fixes ByteSizeValue to serialise correctly (#27702)
* Fixes ByteSizeValue to serialise correctly This fix makes a few fixes to ByteSizeValue to make it possible to perform round-trip serialisation: * Changes wire serialisation to use Zlong methods instead of VLong methods. This is needed because the value `-1` is accepted but previously if `-1` is supplied it cannot be serialised using the wire protocol. * Limits the supplied size to be no more than Long.MAX_VALUE when converted to bytes. Previously values greater than Long.MAX_VALUE bytes were accepted but would be silently interpreted as Long.MAX_VALUE bytes rather than erroring so the user had no idea the value was not being used the way they had intended. I consider this a bug and so fine to include this bug fix in a minor version but I am open to other points of view. * Adds a `getStringRep()` method that can be used when serialising the value to JSON. This will print the bytes value if the size is positive, `”0”` if the size is `0` and `”-1”` if the size is `-1`. * Adds logic to detect fractional values when parsing from a String and emits a deprecation warning in this case. * Modifies hashCode and equals methods to work with long values rather than doubles so they don’t run into precision problems when dealing with large values. Previous to this change the equals method would not detect small differences in the values (e.g. 1-1000 bytes ranges) if the actual values where very large (e.g. PBs). This was due to the values being in the order of 10^18 but doubles only maintaining a precision of ~10^15. Closes #27568 * Fix bytes settings default value to not use fractional values * Fixes test * Addresses review comments * Modifies parsing to preserve unit This should be bwc since in the case that the input is fractional it reverts back to the old method of parsing it to the bytes value. * Addresses more review comments * Fixes tests * Temporarily changes version check to 7.0.0 This will be changed to 6.2 when the fix has been backported
This commit is contained in:
Colin Goodheart-Smithe
GitHub
parent
d26b33dea2
commit
579d1fea57
@ -52,7 +52,6 @@ import java.util.Set;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.IntConsumer;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
@ -1020,7 +1019,7 @@ public class Setting<T> implements ToXContentObject {
|
||||
|
||||
public static Setting<ByteSizeValue> byteSizeSetting(String key, ByteSizeValue defaultValue, ByteSizeValue minValue,
|
||||
ByteSizeValue maxValue, Property... properties) {
|
||||
return byteSizeSetting(key, (s) -> defaultValue.toString(), minValue, maxValue, properties);
|
||||
return byteSizeSetting(key, (s) -> defaultValue.getStringRep(), minValue, maxValue, properties);
|
||||
}
|
||||
|
||||
public static Setting<ByteSizeValue> byteSizeSetting(String key, Function<Settings, String> defaultValue,
|
||||
|
||||
@ -63,6 +63,11 @@ public enum ByteSizeUnit implements Writeable {
|
||||
public long toPB(long size) {
|
||||
return size / (C5 / C0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSuffix() {
|
||||
return "b";
|
||||
}
|
||||
},
|
||||
KB {
|
||||
@Override
|
||||
@ -94,6 +99,11 @@ public enum ByteSizeUnit implements Writeable {
|
||||
public long toPB(long size) {
|
||||
return size / (C5 / C1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSuffix() {
|
||||
return "kb";
|
||||
}
|
||||
},
|
||||
MB {
|
||||
@Override
|
||||
@ -125,6 +135,11 @@ public enum ByteSizeUnit implements Writeable {
|
||||
public long toPB(long size) {
|
||||
return size / (C5 / C2);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSuffix() {
|
||||
return "mb";
|
||||
}
|
||||
},
|
||||
GB {
|
||||
@Override
|
||||
@ -156,6 +171,11 @@ public enum ByteSizeUnit implements Writeable {
|
||||
public long toPB(long size) {
|
||||
return size / (C5 / C3);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSuffix() {
|
||||
return "gb";
|
||||
}
|
||||
},
|
||||
TB {
|
||||
@Override
|
||||
@ -187,6 +207,11 @@ public enum ByteSizeUnit implements Writeable {
|
||||
public long toPB(long size) {
|
||||
return size / (C5 / C4);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSuffix() {
|
||||
return "tb";
|
||||
}
|
||||
},
|
||||
PB {
|
||||
@Override
|
||||
@ -218,6 +243,11 @@ public enum ByteSizeUnit implements Writeable {
|
||||
public long toPB(long size) {
|
||||
return size;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSuffix() {
|
||||
return "pb";
|
||||
}
|
||||
};
|
||||
|
||||
static final long C0 = 1L;
|
||||
@ -258,6 +288,8 @@ public enum ByteSizeUnit implements Writeable {
|
||||
|
||||
public abstract long toPB(long size);
|
||||
|
||||
public abstract String getSuffix();
|
||||
|
||||
@Override
|
||||
public void writeTo(StreamOutput out) throws IOException {
|
||||
out.writeVInt(this.ordinal());
|
||||
|
||||
@ -20,28 +20,42 @@
|
||||
package org.elasticsearch.common.unit;
|
||||
|
||||
import org.elasticsearch.ElasticsearchParseException;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.io.stream.Writeable;
|
||||
import org.elasticsearch.common.logging.DeprecationLogger;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
|
||||
public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue> {
|
||||
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(ByteSizeValue.class));
|
||||
|
||||
private final long size;
|
||||
private final ByteSizeUnit unit;
|
||||
|
||||
public ByteSizeValue(StreamInput in) throws IOException {
|
||||
size = in.readVLong();
|
||||
unit = ByteSizeUnit.BYTES;
|
||||
if (in.getVersion().before(Version.V_7_0_0_alpha1)) {
|
||||
size = in.readVLong();
|
||||
unit = ByteSizeUnit.BYTES;
|
||||
} else {
|
||||
size = in.readZLong();
|
||||
unit = ByteSizeUnit.readFrom(in);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeTo(StreamOutput out) throws IOException {
|
||||
out.writeVLong(getBytes());
|
||||
if (out.getVersion().before(Version.V_7_0_0_alpha1)) {
|
||||
out.writeVLong(getBytes());
|
||||
} else {
|
||||
out.writeZLong(size);
|
||||
unit.writeTo(out);
|
||||
}
|
||||
}
|
||||
|
||||
public ByteSizeValue(long bytes) {
|
||||
@ -49,10 +63,28 @@ public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue> {
|
||||
}
|
||||
|
||||
public ByteSizeValue(long size, ByteSizeUnit unit) {
|
||||
if (size < -1 || (size == -1 && unit != ByteSizeUnit.BYTES)) {
|
||||
throw new IllegalArgumentException("Values less than -1 bytes are not supported: " + size + unit.getSuffix());
|
||||
}
|
||||
if (size > Long.MAX_VALUE / unit.toBytes(1)) {
|
||||
throw new IllegalArgumentException(
|
||||
"Values greater than " + Long.MAX_VALUE + " bytes are not supported: " + size + unit.getSuffix());
|
||||
}
|
||||
this.size = size;
|
||||
this.unit = unit;
|
||||
}
|
||||
|
||||
// For testing
|
||||
long getSize() {
|
||||
return size;
|
||||
}
|
||||
|
||||
// For testing
|
||||
ByteSizeUnit getUnit() {
|
||||
return unit;
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public int bytesAsInt() {
|
||||
long bytes = getBytes();
|
||||
if (bytes > Integer.MAX_VALUE) {
|
||||
@ -105,26 +137,41 @@ public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue> {
|
||||
return ((double) getBytes()) / ByteSizeUnit.C5;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return a string representation of this value which is guaranteed to be
|
||||
* able to be parsed using
|
||||
* {@link #parseBytesSizeValue(String, ByteSizeValue, String)}.
|
||||
* Unlike {@link #toString()} this method will not output fractional
|
||||
* or rounded values so this method should be preferred when
|
||||
* serialising the value to JSON.
|
||||
*/
|
||||
public String getStringRep() {
|
||||
if (size <= 0) {
|
||||
return String.valueOf(size);
|
||||
}
|
||||
return size + unit.getSuffix();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
long bytes = getBytes();
|
||||
double value = bytes;
|
||||
String suffix = "b";
|
||||
String suffix = ByteSizeUnit.BYTES.getSuffix();
|
||||
if (bytes >= ByteSizeUnit.C5) {
|
||||
value = getPbFrac();
|
||||
suffix = "pb";
|
||||
suffix = ByteSizeUnit.PB.getSuffix();
|
||||
} else if (bytes >= ByteSizeUnit.C4) {
|
||||
value = getTbFrac();
|
||||
suffix = "tb";
|
||||
suffix = ByteSizeUnit.TB.getSuffix();
|
||||
} else if (bytes >= ByteSizeUnit.C3) {
|
||||
value = getGbFrac();
|
||||
suffix = "gb";
|
||||
suffix = ByteSizeUnit.GB.getSuffix();
|
||||
} else if (bytes >= ByteSizeUnit.C2) {
|
||||
value = getMbFrac();
|
||||
suffix = "mb";
|
||||
suffix = ByteSizeUnit.MB.getSuffix();
|
||||
} else if (bytes >= ByteSizeUnit.C1) {
|
||||
value = getKbFrac();
|
||||
suffix = "kb";
|
||||
suffix = ByteSizeUnit.KB.getSuffix();
|
||||
}
|
||||
return Strings.format1Decimals(value, suffix);
|
||||
}
|
||||
@ -139,47 +186,64 @@ public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue> {
|
||||
if (sValue == null) {
|
||||
return defaultValue;
|
||||
}
|
||||
long bytes;
|
||||
try {
|
||||
String lowerSValue = sValue.toLowerCase(Locale.ROOT).trim();
|
||||
if (lowerSValue.endsWith("k")) {
|
||||
bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * ByteSizeUnit.C1);
|
||||
} else if (lowerSValue.endsWith("kb")) {
|
||||
bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) * ByteSizeUnit.C1);
|
||||
} else if (lowerSValue.endsWith("m")) {
|
||||
bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * ByteSizeUnit.C2);
|
||||
} else if (lowerSValue.endsWith("mb")) {
|
||||
bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) * ByteSizeUnit.C2);
|
||||
} else if (lowerSValue.endsWith("g")) {
|
||||
bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * ByteSizeUnit.C3);
|
||||
} else if (lowerSValue.endsWith("gb")) {
|
||||
bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) * ByteSizeUnit.C3);
|
||||
} else if (lowerSValue.endsWith("t")) {
|
||||
bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * ByteSizeUnit.C4);
|
||||
} else if (lowerSValue.endsWith("tb")) {
|
||||
bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) * ByteSizeUnit.C4);
|
||||
} else if (lowerSValue.endsWith("p")) {
|
||||
bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * ByteSizeUnit.C5);
|
||||
} else if (lowerSValue.endsWith("pb")) {
|
||||
bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) * ByteSizeUnit.C5);
|
||||
} else if (lowerSValue.endsWith("b")) {
|
||||
bytes = Long.parseLong(lowerSValue.substring(0, lowerSValue.length() - 1).trim());
|
||||
} else if (lowerSValue.equals("-1")) {
|
||||
// Allow this special value to be unit-less:
|
||||
bytes = -1;
|
||||
} else if (lowerSValue.equals("0")) {
|
||||
// Allow this special value to be unit-less:
|
||||
bytes = 0;
|
||||
} else {
|
||||
// Missing units:
|
||||
throw new ElasticsearchParseException(
|
||||
"failed to parse setting [{}] with value [{}] as a size in bytes: unit is missing or unrecognized",
|
||||
settingName, sValue);
|
||||
}
|
||||
} catch (NumberFormatException e) {
|
||||
throw new ElasticsearchParseException("failed to parse [{}]", e, sValue);
|
||||
String lowerSValue = sValue.toLowerCase(Locale.ROOT).trim();
|
||||
if (lowerSValue.endsWith("k")) {
|
||||
return parse(sValue, lowerSValue, "k", ByteSizeUnit.KB, settingName);
|
||||
} else if (lowerSValue.endsWith("kb")) {
|
||||
return parse(sValue, lowerSValue, "kb", ByteSizeUnit.KB, settingName);
|
||||
} else if (lowerSValue.endsWith("m")) {
|
||||
return parse(sValue, lowerSValue, "m", ByteSizeUnit.MB, settingName);
|
||||
} else if (lowerSValue.endsWith("mb")) {
|
||||
return parse(sValue, lowerSValue, "mb", ByteSizeUnit.MB, settingName);
|
||||
} else if (lowerSValue.endsWith("g")) {
|
||||
return parse(sValue, lowerSValue, "g", ByteSizeUnit.GB, settingName);
|
||||
} else if (lowerSValue.endsWith("gb")) {
|
||||
return parse(sValue, lowerSValue, "gb", ByteSizeUnit.GB, settingName);
|
||||
} else if (lowerSValue.endsWith("t")) {
|
||||
return parse(sValue, lowerSValue, "t", ByteSizeUnit.TB, settingName);
|
||||
} else if (lowerSValue.endsWith("tb")) {
|
||||
return parse(sValue, lowerSValue, "tb", ByteSizeUnit.TB, settingName);
|
||||
} else if (lowerSValue.endsWith("p")) {
|
||||
return parse(sValue, lowerSValue, "p", ByteSizeUnit.PB, settingName);
|
||||
} else if (lowerSValue.endsWith("pb")) {
|
||||
return parse(sValue, lowerSValue, "pb", ByteSizeUnit.PB, settingName);
|
||||
} else if (lowerSValue.endsWith("b")) {
|
||||
return new ByteSizeValue(Long.parseLong(lowerSValue.substring(0, lowerSValue.length() - 1).trim()), ByteSizeUnit.BYTES);
|
||||
} else if (lowerSValue.equals("-1")) {
|
||||
// Allow this special value to be unit-less:
|
||||
return new ByteSizeValue(-1, ByteSizeUnit.BYTES);
|
||||
} else if (lowerSValue.equals("0")) {
|
||||
// Allow this special value to be unit-less:
|
||||
return new ByteSizeValue(0, ByteSizeUnit.BYTES);
|
||||
} else {
|
||||
// Missing units:
|
||||
throw new ElasticsearchParseException(
|
||||
"failed to parse setting [{}] with value [{}] as a size in bytes: unit is missing or unrecognized", settingName,
|
||||
sValue);
|
||||
}
|
||||
}
|
||||
|
||||
private static ByteSizeValue parse(final String initialInput, final String normalized, final String suffix, ByteSizeUnit unit,
|
||||
final String settingName) {
|
||||
final String s = normalized.substring(0, normalized.length() - suffix.length()).trim();
|
||||
try {
|
||||
try {
|
||||
return new ByteSizeValue(Long.parseLong(s), unit);
|
||||
} catch (final NumberFormatException e) {
|
||||
try {
|
||||
final double doubleValue = Double.parseDouble(s);
|
||||
DEPRECATION_LOGGER.deprecated(
|
||||
"Fractional bytes values are deprecated. Use non-fractional bytes values instead: [{}] found for setting [{}]",
|
||||
initialInput, settingName);
|
||||
return new ByteSizeValue((long) (doubleValue * unit.toBytes(1)));
|
||||
} catch (final NumberFormatException ignored) {
|
||||
throw new ElasticsearchParseException("failed to parse [{}]", e, initialInput);
|
||||
}
|
||||
}
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ElasticsearchParseException("failed to parse setting [{}] with value [{}] as a size in bytes", e, settingName,
|
||||
initialInput);
|
||||
}
|
||||
return new ByteSizeValue(bytes, ByteSizeUnit.BYTES);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -196,13 +260,13 @@ public class ByteSizeValue implements Writeable, Comparable<ByteSizeValue> {
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Double.hashCode(((double) size) * unit.toBytes(1));
|
||||
return Long.hashCode(size * unit.toBytes(1));
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(ByteSizeValue other) {
|
||||
double thisValue = ((double) size) * unit.toBytes(1);
|
||||
double otherValue = ((double) other.size) * other.unit.toBytes(1);
|
||||
return Double.compare(thisValue, otherValue);
|
||||
long thisValue = size * unit.toBytes(1);
|
||||
long otherValue = other.size * other.unit.toBytes(1);
|
||||
return Long.compare(thisValue, otherValue);
|
||||
}
|
||||
}
|
||||
|
||||
@ -311,9 +311,11 @@ public class NodeStatsTests extends ESTestCase {
|
||||
for (int i = 0; i < 3; i++) {
|
||||
loadAverages[i] = randomBoolean() ? randomDouble() : -1;
|
||||
}
|
||||
long memTotal = randomNonNegativeLong();
|
||||
long swapTotal = randomNonNegativeLong();
|
||||
osStats = new OsStats(System.currentTimeMillis(), new OsStats.Cpu(randomShort(), loadAverages),
|
||||
new OsStats.Mem(randomLong(), randomLong()),
|
||||
new OsStats.Swap(randomLong(), randomLong()),
|
||||
new OsStats.Mem(memTotal, randomLongBetween(0, memTotal)),
|
||||
new OsStats.Swap(swapTotal, randomLongBetween(0, swapTotal)),
|
||||
new OsStats.Cgroup(
|
||||
randomAlphaOfLength(8),
|
||||
randomNonNegativeLong(),
|
||||
|
||||
@ -23,15 +23,12 @@ import org.elasticsearch.Version;
|
||||
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
|
||||
import org.elasticsearch.action.admin.indices.stats.CommonStats;
|
||||
import org.elasticsearch.action.admin.indices.stats.ShardStats;
|
||||
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
||||
import org.elasticsearch.cluster.metadata.MetaData;
|
||||
import org.elasticsearch.cluster.node.DiscoveryNode;
|
||||
import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
|
||||
import org.elasticsearch.cluster.routing.ShardRouting;
|
||||
import org.elasticsearch.cluster.routing.ShardRoutingHelper;
|
||||
import org.elasticsearch.cluster.routing.UnassignedInfo;
|
||||
import org.elasticsearch.common.collect.ImmutableOpenMap;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.shard.ShardId;
|
||||
import org.elasticsearch.index.shard.ShardPath;
|
||||
@ -184,12 +181,12 @@ public class DiskUsageTests extends ESTestCase {
|
||||
new FsInfo.Path("/most", "/dev/sdc", 300, 290, 280),
|
||||
};
|
||||
FsInfo.Path[] node2FSInfo = new FsInfo.Path[] {
|
||||
new FsInfo.Path("/least_most", "/dev/sda", -2, -1, -1),
|
||||
new FsInfo.Path("/least_most", "/dev/sda", -1, -1, -1),
|
||||
};
|
||||
|
||||
FsInfo.Path[] node3FSInfo = new FsInfo.Path[] {
|
||||
new FsInfo.Path("/most", "/dev/sda", 100, 90, 70),
|
||||
new FsInfo.Path("/least", "/dev/sda", 10, -8, 0),
|
||||
new FsInfo.Path("/least", "/dev/sda", 10, -1, 0),
|
||||
};
|
||||
List<NodeStats> nodeStats = Arrays.asList(
|
||||
new NodeStats(new DiscoveryNode("node_1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0,
|
||||
|
||||
@ -20,9 +20,9 @@
|
||||
package org.elasticsearch.common.unit;
|
||||
|
||||
import org.elasticsearch.ElasticsearchParseException;
|
||||
import org.elasticsearch.common.io.stream.BytesStreamOutput;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.io.stream.Writeable.Reader;
|
||||
import org.elasticsearch.test.AbstractWireSerializingTestCase;
|
||||
import org.hamcrest.MatcherAssert;
|
||||
|
||||
import java.io.IOException;
|
||||
@ -31,7 +31,7 @@ import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
|
||||
public class ByteSizeValueTests extends ESTestCase {
|
||||
public class ByteSizeValueTests extends AbstractWireSerializingTestCase<ByteSizeValue> {
|
||||
public void testActualPeta() {
|
||||
MatcherAssert.assertThat(new ByteSizeValue(4, ByteSizeUnit.PB).getBytes(), equalTo(4503599627370496L));
|
||||
}
|
||||
@ -150,17 +150,17 @@ public class ByteSizeValueTests extends ESTestCase {
|
||||
}
|
||||
|
||||
public void testCompareEquality() {
|
||||
long firstRandom = randomNonNegativeLong();
|
||||
ByteSizeUnit randomUnit = randomFrom(ByteSizeUnit.values());
|
||||
long firstRandom = randomNonNegativeLong() / randomUnit.toBytes(1);
|
||||
ByteSizeValue firstByteValue = new ByteSizeValue(firstRandom, randomUnit);
|
||||
ByteSizeValue secondByteValue = new ByteSizeValue(firstRandom, randomUnit);
|
||||
assertEquals(0, firstByteValue.compareTo(secondByteValue));
|
||||
}
|
||||
|
||||
public void testCompareValue() {
|
||||
long firstRandom = randomNonNegativeLong();
|
||||
long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomNonNegativeLong);
|
||||
ByteSizeUnit unit = randomFrom(ByteSizeUnit.values());
|
||||
long firstRandom = randomNonNegativeLong() / unit.toBytes(1);
|
||||
long secondRandom = randomValueOtherThan(firstRandom, () -> randomNonNegativeLong() / unit.toBytes(1));
|
||||
ByteSizeValue firstByteValue = new ByteSizeValue(firstRandom, unit);
|
||||
ByteSizeValue secondByteValue = new ByteSizeValue(secondRandom, unit);
|
||||
assertEquals(firstRandom > secondRandom, firstByteValue.compareTo(secondByteValue) > 0);
|
||||
@ -168,7 +168,7 @@ public class ByteSizeValueTests extends ESTestCase {
|
||||
}
|
||||
|
||||
public void testCompareUnits() {
|
||||
long number = randomNonNegativeLong();
|
||||
long number = randomNonNegativeLong() / ByteSizeUnit.PB.toBytes(1);
|
||||
ByteSizeUnit randomUnit = randomValueOtherThan(ByteSizeUnit.PB, ()->randomFrom(ByteSizeUnit.values()));
|
||||
ByteSizeValue firstByteValue = new ByteSizeValue(number, randomUnit);
|
||||
ByteSizeValue secondByteValue = new ByteSizeValue(number, ByteSizeUnit.PB);
|
||||
@ -176,10 +176,25 @@ public class ByteSizeValueTests extends ESTestCase {
|
||||
assertTrue(secondByteValue.compareTo(firstByteValue) > 0);
|
||||
}
|
||||
|
||||
public void testEdgeCompare() {
|
||||
ByteSizeValue maxLongValuePB = new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.PB);
|
||||
ByteSizeValue maxLongValueB = new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES);
|
||||
assertTrue(maxLongValuePB.compareTo(maxLongValueB) > 0);
|
||||
public void testOutOfRange() {
|
||||
// Make sure a value of > Long.MAX_VALUE bytes throws an exception
|
||||
ByteSizeUnit unit = randomValueOtherThan(ByteSizeUnit.BYTES, () -> randomFrom(ByteSizeUnit.values()));
|
||||
long size = (long) randomDouble() * unit.toBytes(1) + (Long.MAX_VALUE - unit.toBytes(1));
|
||||
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new ByteSizeValue(size, unit));
|
||||
assertEquals("Values greater than " + Long.MAX_VALUE + " bytes are not supported: " + size + unit.getSuffix(),
|
||||
exception.getMessage());
|
||||
|
||||
// Make sure for units other than BYTES a size of -1 throws an exception
|
||||
ByteSizeUnit unit2 = randomValueOtherThan(ByteSizeUnit.BYTES, () -> randomFrom(ByteSizeUnit.values()));
|
||||
long size2 = -1L;
|
||||
exception = expectThrows(IllegalArgumentException.class, () -> new ByteSizeValue(size2, unit2));
|
||||
assertEquals("Values less than -1 bytes are not supported: " + size2 + unit2.getSuffix(), exception.getMessage());
|
||||
|
||||
// Make sure for any unit a size < -1 throws an exception
|
||||
ByteSizeUnit unit3 = randomFrom(ByteSizeUnit.values());
|
||||
long size3 = -1L * randomNonNegativeLong() - 1L;
|
||||
exception = expectThrows(IllegalArgumentException.class, () -> new ByteSizeValue(size3, unit3));
|
||||
assertEquals("Values less than -1 bytes are not supported: " + size3 + unit3.getSuffix(), exception.getMessage());
|
||||
}
|
||||
|
||||
public void testConversionHashCode() {
|
||||
@ -188,14 +203,114 @@ public class ByteSizeValueTests extends ESTestCase {
|
||||
assertEquals(firstValue.hashCode(), secondValue.hashCode());
|
||||
}
|
||||
|
||||
public void testSerialization() throws IOException {
|
||||
ByteSizeValue byteSizeValue = new ByteSizeValue(randomNonNegativeLong(), randomFrom(ByteSizeUnit.values()));
|
||||
try (BytesStreamOutput out = new BytesStreamOutput()) {
|
||||
byteSizeValue.writeTo(out);
|
||||
try (StreamInput in = out.bytes().streamInput()) {
|
||||
ByteSizeValue deserializedByteSizeValue = new ByteSizeValue(in);
|
||||
assertEquals(byteSizeValue.getBytes(), deserializedByteSizeValue.getBytes());
|
||||
@Override
|
||||
protected ByteSizeValue createTestInstance() {
|
||||
if (randomBoolean()) {
|
||||
ByteSizeUnit unit = randomFrom(ByteSizeUnit.values());
|
||||
long size = randomNonNegativeLong() / unit.toBytes(1);
|
||||
if (size >= Long.MAX_VALUE / unit.toBytes(1)) {
|
||||
throw new AssertionError();
|
||||
}
|
||||
return new ByteSizeValue(size, unit);
|
||||
} else {
|
||||
return new ByteSizeValue(randomNonNegativeLong());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Reader<ByteSizeValue> instanceReader() {
|
||||
return ByteSizeValue::new;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ByteSizeValue mutateInstance(ByteSizeValue instance) throws IOException {
|
||||
long size = instance.getSize();
|
||||
ByteSizeUnit unit = instance.getUnit();
|
||||
switch (between(0, 1)) {
|
||||
case 0:
|
||||
long unitBytes = unit.toBytes(1);
|
||||
size = randomValueOtherThan(size, () -> randomNonNegativeLong() / unitBytes);
|
||||
break;
|
||||
case 1:
|
||||
unit = randomValueOtherThan(unit, () -> randomFrom(ByteSizeUnit.values()));
|
||||
long newUnitBytes = unit.toBytes(1);
|
||||
if (size >= Long.MAX_VALUE / newUnitBytes) {
|
||||
size = randomValueOtherThan(size, () -> randomNonNegativeLong() / newUnitBytes);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
throw new AssertionError("Invalid randomisation branch");
|
||||
}
|
||||
return new ByteSizeValue(size, unit);
|
||||
}
|
||||
|
||||
public void testParse() {
|
||||
for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
|
||||
ByteSizeValue original = createTestInstance();
|
||||
String serialised = original.getStringRep();
|
||||
ByteSizeValue copy = ByteSizeValue.parseBytesSizeValue(serialised, "test");
|
||||
assertEquals(original, copy);
|
||||
assertEquals(serialised, copy.getStringRep());
|
||||
}
|
||||
}
|
||||
|
||||
public void testParseInvalidValue() {
|
||||
ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class,
|
||||
() -> ByteSizeValue.parseBytesSizeValue("-6mb", "test_setting"));
|
||||
assertEquals("failed to parse setting [test_setting] with value [-6mb] as a size in bytes", exception.getMessage());
|
||||
assertNotNull(exception.getCause());
|
||||
assertEquals(IllegalArgumentException.class, exception.getCause().getClass());
|
||||
}
|
||||
|
||||
public void testParseDefaultValue() {
|
||||
ByteSizeValue defaultValue = createTestInstance();
|
||||
assertEquals(defaultValue, ByteSizeValue.parseBytesSizeValue(null, defaultValue, "test"));
|
||||
}
|
||||
|
||||
public void testParseSpecialValues() throws IOException {
|
||||
ByteSizeValue instance = new ByteSizeValue(-1);
|
||||
assertEquals(instance, ByteSizeValue.parseBytesSizeValue(instance.getStringRep(), null, "test"));
|
||||
assertSerialization(instance);
|
||||
|
||||
instance = new ByteSizeValue(0);
|
||||
assertEquals(instance, ByteSizeValue.parseBytesSizeValue(instance.getStringRep(), null, "test"));
|
||||
assertSerialization(instance);
|
||||
}
|
||||
|
||||
public void testParseInvalidNumber() throws IOException {
|
||||
ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class,
|
||||
() -> ByteSizeValue.parseBytesSizeValue("notANumber", "test"));
|
||||
assertEquals("failed to parse setting [test] with value [notANumber] as a size in bytes: unit is missing or unrecognized",
|
||||
exception.getMessage());
|
||||
|
||||
exception = expectThrows(ElasticsearchParseException.class, () -> ByteSizeValue.parseBytesSizeValue("notANumberMB", "test"));
|
||||
assertEquals("failed to parse [notANumberMB]", exception.getMessage());
|
||||
}
|
||||
|
||||
public void testParseFractionalNumber() throws IOException {
|
||||
ByteSizeUnit unit = randomValueOtherThan(ByteSizeUnit.BYTES, () -> randomFrom(ByteSizeUnit.values()));
|
||||
String fractionalValue = "23.5" + unit.getSuffix();
|
||||
ByteSizeValue instance = ByteSizeValue.parseBytesSizeValue(fractionalValue, "test");
|
||||
assertEquals(fractionalValue, instance.toString());
|
||||
assertWarnings("Fractional bytes values are deprecated. Use non-fractional bytes values instead: [" + fractionalValue
|
||||
+ "] found for setting [test]");
|
||||
}
|
||||
|
||||
public void testGetBytesAsInt() {
|
||||
for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
|
||||
ByteSizeValue instance = new ByteSizeValue(randomIntBetween(1, 1000), randomFrom(ByteSizeUnit.values()));
|
||||
long bytesValue = instance.getBytes();
|
||||
if (bytesValue > Integer.MAX_VALUE) {
|
||||
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> instance.bytesAsInt());
|
||||
assertEquals("size [" + instance.toString() + "] is bigger than max int", exception.getMessage());
|
||||
} else {
|
||||
assertEquals((int) bytesValue, instance.bytesAsInt());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void testOldSerialisation() throws IOException {
|
||||
ByteSizeValue original = createTestInstance();
|
||||
assertSerialization(original, randomFrom(Version.V_5_6_4, Version.V_5_6_5, Version.V_6_0_0, Version.V_6_0_1, Version.V_6_1_0));
|
||||
}
|
||||
}
|
||||
|
||||
@ -408,19 +408,21 @@ public class IndexSettingsTests extends ESTestCase {
|
||||
|
||||
public void testTranslogFlushSizeThreshold() {
|
||||
ByteSizeValue translogFlushThresholdSize = new ByteSizeValue(Math.abs(randomInt()));
|
||||
ByteSizeValue actualValue = ByteSizeValue.parseBytesSizeValue(translogFlushThresholdSize.toString(),
|
||||
ByteSizeValue actualValue = ByteSizeValue.parseBytesSizeValue(translogFlushThresholdSize.getBytes() + "B",
|
||||
IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey());
|
||||
IndexMetaData metaData = newIndexMeta("index", Settings.builder()
|
||||
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), translogFlushThresholdSize.toString())
|
||||
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), translogFlushThresholdSize.getBytes() + "B")
|
||||
.build());
|
||||
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
|
||||
assertEquals(actualValue, settings.getFlushThresholdSize());
|
||||
ByteSizeValue newTranslogFlushThresholdSize = new ByteSizeValue(Math.abs(randomInt()));
|
||||
ByteSizeValue actualNewTranslogFlushThresholdSize = ByteSizeValue.parseBytesSizeValue(newTranslogFlushThresholdSize.toString(),
|
||||
ByteSizeValue actualNewTranslogFlushThresholdSize = ByteSizeValue.parseBytesSizeValue(
|
||||
newTranslogFlushThresholdSize.getBytes() + "B",
|
||||
IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey());
|
||||
settings.updateIndexMetaData(newIndexMeta("index", Settings.builder()
|
||||
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), newTranslogFlushThresholdSize.toString()).build()));
|
||||
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), newTranslogFlushThresholdSize.getBytes() + "B")
|
||||
.build()));
|
||||
assertEquals(actualNewTranslogFlushThresholdSize, settings.getFlushThresholdSize());
|
||||
}
|
||||
|
||||
@ -428,20 +430,20 @@ public class IndexSettingsTests extends ESTestCase {
|
||||
final ByteSizeValue size = new ByteSizeValue(Math.abs(randomInt()));
|
||||
final String key = IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING.getKey();
|
||||
final ByteSizeValue actualValue =
|
||||
ByteSizeValue.parseBytesSizeValue(size.toString(), key);
|
||||
ByteSizeValue.parseBytesSizeValue(size.getBytes() + "B", key);
|
||||
final IndexMetaData metaData =
|
||||
newIndexMeta(
|
||||
"index",
|
||||
Settings.builder()
|
||||
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.put(key, size.toString())
|
||||
.put(key, size.getBytes() + "B")
|
||||
.build());
|
||||
final IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
|
||||
assertEquals(actualValue, settings.getGenerationThresholdSize());
|
||||
final ByteSizeValue newSize = new ByteSizeValue(Math.abs(randomInt()));
|
||||
final ByteSizeValue actual = ByteSizeValue.parseBytesSizeValue(newSize.toString(), key);
|
||||
final ByteSizeValue actual = ByteSizeValue.parseBytesSizeValue(newSize.getBytes() + "B", key);
|
||||
settings.updateIndexMetaData(
|
||||
newIndexMeta("index", Settings.builder().put(key, newSize.toString()).build()));
|
||||
newIndexMeta("index", Settings.builder().put(key, newSize.getBytes() + "B").build()));
|
||||
assertEquals(actual, settings.getGenerationThresholdSize());
|
||||
}
|
||||
|
||||
|
||||
@ -35,8 +35,8 @@ import org.elasticsearch.index.shard.IndexShardTestCase;
|
||||
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
|
||||
import org.elasticsearch.indices.recovery.RecoveryState;
|
||||
import org.elasticsearch.test.ESSingleNodeTestCase;
|
||||
import org.elasticsearch.threadpool.ThreadPool;
|
||||
import org.elasticsearch.threadpool.Scheduler.Cancellable;
|
||||
import org.elasticsearch.threadpool.ThreadPool;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
@ -250,7 +250,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
|
||||
Exception e = expectThrows(IllegalArgumentException.class,
|
||||
() -> new MockController(Settings.builder()
|
||||
.put("indices.memory.min_index_buffer_size", "-6mb").build()));
|
||||
assertEquals("Failed to parse value [-6mb] for setting [indices.memory.min_index_buffer_size] must be >= 0b", e.getMessage());
|
||||
assertEquals("failed to parse setting [indices.memory.min_index_buffer_size] with value [-6mb] as a size in bytes", e.getMessage());
|
||||
|
||||
}
|
||||
|
||||
@ -274,7 +274,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
|
||||
Exception e = expectThrows(IllegalArgumentException.class,
|
||||
() -> new MockController(Settings.builder()
|
||||
.put("indices.memory.max_index_buffer_size", "-6mb").build()));
|
||||
assertEquals("Failed to parse value [-6mb] for setting [indices.memory.max_index_buffer_size] must be >= -1b", e.getMessage());
|
||||
assertEquals("failed to parse setting [indices.memory.max_index_buffer_size] with value [-6mb] as a size in bytes", e.getMessage());
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -34,8 +34,10 @@ public class OsStatsTests extends ESTestCase {
|
||||
loadAverages[i] = randomDouble();
|
||||
}
|
||||
OsStats.Cpu cpu = new OsStats.Cpu(randomShort(), loadAverages);
|
||||
OsStats.Mem mem = new OsStats.Mem(randomLong(), randomLong());
|
||||
OsStats.Swap swap = new OsStats.Swap(randomLong(), randomLong());
|
||||
long memTotal = randomNonNegativeLong();
|
||||
OsStats.Mem mem = new OsStats.Mem(memTotal, randomLongBetween(0, memTotal));
|
||||
long swapTotal = randomNonNegativeLong();
|
||||
OsStats.Swap swap = new OsStats.Swap(swapTotal, randomLongBetween(0, swapTotal));
|
||||
OsStats.Cgroup cgroup = new OsStats.Cgroup(
|
||||
randomAlphaOfLength(8),
|
||||
randomNonNegativeLong(),
|
||||
|
||||
@ -19,10 +19,9 @@
|
||||
|
||||
package org.elasticsearch.repositories.s3;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import com.amazonaws.services.s3.AbstractAmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
|
||||
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
|
||||
import org.elasticsearch.common.component.AbstractLifecycleComponent;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
@ -33,6 +32,8 @@ import org.elasticsearch.repositories.RepositoryException;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.hamcrest.Matchers;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
|
||||
public class S3RepositoryTests extends ESTestCase {
|
||||
@ -69,23 +70,23 @@ public class S3RepositoryTests extends ESTestCase {
|
||||
assertValidBuffer(5, 5);
|
||||
// buffer < 5mb should fail
|
||||
assertInvalidBuffer(4, 10, IllegalArgumentException.class,
|
||||
"Failed to parse value [4mb] for setting [buffer_size] must be >= 5mb");
|
||||
"Failed to parse value [4mb] for setting [buffer_size] must be >= 5mb");
|
||||
// chunk > 5tb should fail
|
||||
assertInvalidBuffer(5, 6000000, IllegalArgumentException.class,
|
||||
"Failed to parse value [5.7tb] for setting [chunk_size] must be <= 5tb");
|
||||
"Failed to parse value [6000000mb] for setting [chunk_size] must be <= 5tb");
|
||||
}
|
||||
|
||||
private void assertValidBuffer(long bufferMB, long chunkMB) throws IOException {
|
||||
RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.builder()
|
||||
.put(S3Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB))
|
||||
.put(S3Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB)).build());
|
||||
.put(S3Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB).getStringRep())
|
||||
.put(S3Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB).getStringRep()).build());
|
||||
new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service());
|
||||
}
|
||||
|
||||
private void assertInvalidBuffer(int bufferMB, int chunkMB, Class<? extends Exception> clazz, String msg) throws IOException {
|
||||
RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.builder()
|
||||
.put(S3Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB))
|
||||
.put(S3Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB)).build());
|
||||
.put(S3Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB).getStringRep())
|
||||
.put(S3Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB).getStringRep()).build());
|
||||
|
||||
Exception e = expectThrows(clazz, () -> new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY,
|
||||
new DummyS3Service()));
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user