diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/IPv4AddressBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/IPv4AddressBenchmark.java index 43263c6b938..156d54c03c4 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/IPv4AddressBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/IPv4AddressBenchmark.java @@ -20,6 +20,7 @@ package org.apache.druid.benchmark; import com.google.common.net.InetAddresses; +import inet.ipaddr.AddressStringException; import inet.ipaddr.IPAddress; import inet.ipaddr.IPAddressString; import inet.ipaddr.ipv4.IPv4Address; @@ -38,11 +39,13 @@ import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; +import javax.annotation.Nullable; import java.net.Inet4Address; import java.net.InetAddress; import java.util.ArrayList; @@ -55,28 +58,31 @@ import java.util.regex.Pattern; @State(Scope.Benchmark) @Fork(value = 1) -@Warmup(iterations = 5) +@Warmup(iterations = 3) @Measurement(iterations = 5) @BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.NANOSECONDS) +@OutputTimeUnit(TimeUnit.MILLISECONDS) public class IPv4AddressBenchmark { private static final Pattern IPV4_PATTERN = Pattern.compile( "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$" ); // Different representations of IPv4 addresses. - private static List IPV4_ADDRESS_STRS; - private static List IPV4_ADDRESSES; - private static List INET4_ADDRESSES; - private static List IPV4_ADDRESS_LONGS; - private static List IPV4_SUBNETS; - @Param({"10", "100", "1000"}) + private List inputStrings; + private List inputLongs; + private List addresses; + private List inet4Addresses; + + @Param({"100000"}) public int numOfAddresses; - static boolean isValidAddress(String string) - { - return string != null && IPV4_PATTERN.matcher(string).matches(); - } + @Param({"16", "31"}) + public int prefixRange; + + SubnetUtils.SubnetInfo subnetInfo; + IPAddressString subnetString; + IPAddress subnetBlock; + public static void main(String[] args) throws RunnerException { @@ -90,160 +96,178 @@ public class IPv4AddressBenchmark @Setup public void setUp() { - IPV4_ADDRESS_STRS = new ArrayList<>(numOfAddresses); - IPV4_ADDRESSES = new ArrayList<>(numOfAddresses); - INET4_ADDRESSES = new ArrayList<>(numOfAddresses); - IPV4_SUBNETS = new ArrayList<>(numOfAddresses); - IPV4_ADDRESS_LONGS = new ArrayList<>(numOfAddresses); + inputStrings = new ArrayList<>(numOfAddresses); + addresses = new ArrayList<>(numOfAddresses); + inet4Addresses = new ArrayList<>(numOfAddresses); + inputLongs = new ArrayList<>(numOfAddresses); + + Random r = ThreadLocalRandom.current(); + String subnetAddress = generateIpAddressString(r) + "/" + prefixRange; + + try { + subnetString = new IPAddressString(subnetAddress); + subnetBlock = subnetString.toAddress().toPrefixBlock(); + subnetInfo = getSubnetInfo(subnetAddress); + } + catch (AddressStringException e) { + throw new RuntimeException(e); + } for (int i = 0; i < numOfAddresses; i++) { - Random r = ThreadLocalRandom.current(); - String genIpAddress = r.nextInt(256) + "." + r.nextInt(256) + "." + r.nextInt(256) + "." + r.nextInt(256); + String genIpAddress = generateIpAddressString(r); IPAddressString ipAddressString = new IPAddressString(genIpAddress); - IPAddress prefixBlock = ipAddressString.getAddress().applyPrefixLength(r.nextInt(32)).toPrefixBlock(); - IPV4_ADDRESS_STRS.add(ipAddressString.toString()); - IPV4_SUBNETS.add(prefixBlock.toString()); + inputStrings.add(ipAddressString.toString()); IPv4Address iPv4Address = ipAddressString.getAddress().toIPv4(); - IPV4_ADDRESSES.add(iPv4Address); - INET4_ADDRESSES.add(iPv4Address.toInetAddress()); - IPV4_ADDRESS_LONGS.add(iPv4Address.longValue()); + addresses.add(iPv4Address); + inet4Addresses.add(iPv4Address.toInetAddress()); + inputLongs.add(iPv4Address.longValue()); } } @Benchmark @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public void stringContainsUsingIpAddr() + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void stringContainsUsingIpAddrPrefixContains(Blackhole blackhole) { - for (int i = 0; i < IPV4_ADDRESS_STRS.size(); i++) { - String v4Subnet = IPV4_SUBNETS.get(i); - String v4Address = IPV4_ADDRESS_STRS.get(i); - - IPAddressString subnetString = new IPAddressString(v4Subnet); - IPv4Address iPv4Address = IPv4AddressExprUtils.parse(v4Address); - if (iPv4Address != null) { - subnetString.contains(iPv4Address.toAddressString()); - } + for (String v4Address : inputStrings) { + final IPAddressString iPv4Address = IPv4AddressExprUtils.parseString(v4Address); + blackhole.consume(iPv4Address != null && subnetString.prefixContains(iPv4Address)); } } @Benchmark @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public void stringContainsUsingSubnetUtils() throws IAE + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void stringContainsUsingIpAddrPrefixContainsTooManyRoundTrips(Blackhole blackhole) { - for (int i = 0; i < IPV4_ADDRESS_STRS.size(); i++) { - String v4Subnet = IPV4_SUBNETS.get(i); - String v4Address = IPV4_ADDRESS_STRS.get(i); - - SubnetUtils.SubnetInfo subnetInfo = getSubnetInfo(v4Subnet); - if (isValidAddress(v4Address)) { - subnetInfo.isInRange(v4Address); - } + for (String v4Address : inputStrings) { + final IPv4Address iPv4Address = IPv4AddressExprUtils.parse(v4Address); + blackhole.consume(iPv4Address != null && subnetString.prefixContains(iPv4Address.toAddressString())); } } @Benchmark @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public void parseLongUsingIpAddr() + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void stringContainsUsingSubnetUtils(Blackhole blackhole) throws IAE { - for (Long v4Address : IPV4_ADDRESS_LONGS) { - IPv4AddressExprUtils.parse(v4Address); - + for (String v4Address : inputStrings) { + blackhole.consume(isValidAddress(v4Address) && subnetInfo.isInRange(v4Address)); } } - private Inet4Address parseUsingSubnetUtils(long longValue) - { - if (IPv4AddressExprUtils.overflowsUnsignedInt(longValue)) { - return InetAddresses.fromInteger((int) longValue); - } - return null; - } - @Benchmark @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public void parseLongUsingSubnetUtils() + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void parseLongUsingIpAddr(Blackhole blackhole) { - for (Long v4Address : IPV4_ADDRESS_LONGS) { - parseUsingSubnetUtils(v4Address); + for (Long v4Address : inputLongs) { + blackhole.consume(IPv4AddressExprUtils.parse(v4Address)); } } - private long toLongUsingSubnetUtils(Inet4Address address) + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void parseLongUsingSubnetUtils(Blackhole blackhole) + { + for (Long v4Address : inputLongs) { + blackhole.consume(parseUsingSubnetUtils(v4Address)); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void toLongUsingSubnetUtils(Blackhole blackhole) + { + for (Inet4Address v4InetAddress : inet4Addresses) { + blackhole.consume(toLongUsingSubnetUtils(v4InetAddress)); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void toLongUsingIpAddr(Blackhole blackhole) + { + for (IPv4Address v4Address : addresses) { + blackhole.consume(IPv4AddressExprUtils.toLong(v4Address)); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void longContainsUsingSubnetUtils(Blackhole blackhole) + { + for (long v4Long : inputLongs) { + blackhole.consume(!IPv4AddressExprUtils.overflowsUnsignedInt(v4Long) && subnetInfo.isInRange((int) v4Long)); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void longContainsUsingIpAddr(Blackhole blackhole) + { + for (long v4Long : inputLongs) { + final IPv4Address iPv4Address = IPv4AddressExprUtils.parse(v4Long); + blackhole.consume(iPv4Address != null && subnetBlock.contains(iPv4Address)); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void longContainsUsingIpAddrWithStrings(Blackhole blackhole) + { + for (long v4Long : inputLongs) { + final IPv4Address iPv4Address = IPv4AddressExprUtils.parse(v4Long); + blackhole.consume(iPv4Address != null && subnetString.prefixContains(iPv4Address.toAddressString())); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void parseStringUsingIpAddr(Blackhole blackhole) + { + for (String ipv4Addr : inputStrings) { + blackhole.consume(IPv4AddressExprUtils.parse(ipv4Addr)); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void parseStringUsingIpAddrString(Blackhole blackhole) + { + for (String ipv4Addr : inputStrings) { + blackhole.consume(IPv4AddressExprUtils.parseString(ipv4Addr)); + } + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void parseStringUsingSubnetUtils(Blackhole blackhole) + { + for (String ipv4Addr : inputStrings) { + blackhole.consume(parseUsingSubnetUtils(ipv4Addr)); + } + } + + private static long toLongUsingSubnetUtils(Inet4Address address) { int value = InetAddresses.coerceToInteger(address); return Integer.toUnsignedLong(value); } - @Benchmark - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public void toLongUsingSubnetUtils() - { - for (Inet4Address v4InetAddress : INET4_ADDRESSES) { - toLongUsingSubnetUtils(v4InetAddress); - } - } - - @Benchmark - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public void toLongUsingIpAddr() - { - for (IPv4Address v4Address : IPV4_ADDRESSES) { - IPv4AddressExprUtils.toLong(v4Address); - } - } - - @Benchmark - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public void longContainsUsingSubnetUtils() - { - for (int i = 0; i < IPV4_ADDRESS_LONGS.size(); i++) { - long v4Long = IPV4_ADDRESS_LONGS.get(i); - String v4Subnet = IPV4_SUBNETS.get(i); - SubnetUtils.SubnetInfo subnetInfo = getSubnetInfo(v4Subnet); - - if (!IPv4AddressExprUtils.overflowsUnsignedInt(v4Long)) { - subnetInfo.isInRange((int) v4Long); - } - } - } - - @Benchmark - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public void longContainsUsingIpAddr() - { - for (int i = 0; i < IPV4_ADDRESS_LONGS.size(); i++) { - long v4Long = IPV4_ADDRESS_LONGS.get(i); - String v4Subnet = IPV4_SUBNETS.get(i); - - IPv4Address iPv4Address = IPv4AddressExprUtils.parse(v4Long); - IPAddressString subnetString = new IPAddressString(v4Subnet); - if (iPv4Address != null) { - subnetString.contains(iPv4Address.toAddressString()); - } - } - } - - @Benchmark - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public void parseStringUsingIpAddr() - { - for (String ipv4Addr : IPV4_ADDRESS_STRS) { - IPv4AddressExprUtils.parse(ipv4Addr); - } - } - - private Inet4Address parseUsingSubnetUtils(String string) + @Nullable + private static Inet4Address parseUsingSubnetUtils(String string) { if (isValidAddress(string)) { InetAddress address = InetAddresses.forString(string); @@ -254,17 +278,15 @@ public class IPv4AddressBenchmark return null; } - @Benchmark - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public void parseStringUsingSubnetUtils() + private static Inet4Address parseUsingSubnetUtils(long longValue) { - for (String ipv4Addr : IPV4_ADDRESS_STRS) { - parseUsingSubnetUtils(ipv4Addr); + if (IPv4AddressExprUtils.overflowsUnsignedInt(longValue)) { + return InetAddresses.fromInteger((int) longValue); } + return null; } - private SubnetUtils.SubnetInfo getSubnetInfo(String subnet) + private static SubnetUtils.SubnetInfo getSubnetInfo(String subnet) { SubnetUtils subnetUtils; try { @@ -276,4 +298,15 @@ public class IPv4AddressBenchmark subnetUtils.setInclusiveHostCount(true); // make network and broadcast addresses match return subnetUtils.getInfo(); } + + + static boolean isValidAddress(@Nullable String string) + { + return string != null && IPV4_PATTERN.matcher(string).matches(); + } + + private static String generateIpAddressString(Random r) + { + return r.nextInt(256) + "." + r.nextInt(256) + "." + r.nextInt(256) + "." + r.nextInt(256); + } } diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java index f91015d8aa2..12190375b86 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressExprUtils.java @@ -68,6 +68,16 @@ public class IPv4AddressExprUtils return null; } + @Nullable + public static IPAddressString parseString(@Nullable String string) + { + IPAddressString ipAddressString = new IPAddressString(string, IPV4_ADDRESS_PARAMS); + if (ipAddressString.isIPv4()) { + return ipAddressString; + } + return null; + } + /** * @return IPv4 address if the supplied integer is a valid IPv4 integer number. */ diff --git a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java index 33d991aa1d0..33273b2de7b 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java +++ b/processing/src/main/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacro.java @@ -19,6 +19,8 @@ package org.apache.druid.query.expression; +import inet.ipaddr.AddressStringException; +import inet.ipaddr.IPAddress; import inet.ipaddr.IPAddressString; import inet.ipaddr.ipv4.IPv4Address; import org.apache.druid.java.util.common.IAE; @@ -71,71 +73,77 @@ public class IPv4AddressMatchExprMacro implements ExprMacroTable.ExprMacro throw new IAE(ExprUtils.createErrMsg(name(), "must have 2 arguments")); } - IPAddressString subnetInfo = getSubnetInfo(args); - Expr arg = args.get(0); + try { + final Expr arg = args.get(0); + // we use 'blockString' for string matching with 'prefixContains' because we parse them into IPAddressString + // for longs, we convert into a prefix block use 'block' and 'contains' so avoid converting to IPAddressString + final IPAddressString blockString = getSubnetInfo(args); + final IPAddress block = blockString.toAddress().toPrefixBlock(); - class IPv4AddressMatchExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr - { - private final IPAddressString subnetString; - - private IPv4AddressMatchExpr(Expr arg, IPAddressString subnetString) + class IPv4AddressMatchExpr extends ExprMacroTable.BaseScalarUnivariateMacroFunctionExpr { - super(FN_NAME, arg); - this.subnetString = subnetString; - } - - @Nonnull - @Override - public ExprEval eval(final ObjectBinding bindings) - { - ExprEval eval = arg.eval(bindings); - boolean match; - switch (eval.type().getType()) { - case STRING: - match = isStringMatch(eval.asString()); - break; - case LONG: - match = !eval.isNumericNull() && isLongMatch(eval.asLong()); - break; - default: - match = false; + private IPv4AddressMatchExpr(Expr arg) + { + super(FN_NAME, arg); + } + + @Nonnull + @Override + public ExprEval eval(final ObjectBinding bindings) + { + ExprEval eval = arg.eval(bindings); + boolean match; + switch (eval.type().getType()) { + case STRING: + match = isStringMatch(eval.asString()); + break; + case LONG: + match = !eval.isNumericNull() && isLongMatch(eval.asLong()); + break; + default: + match = false; + } + return ExprEval.ofLongBoolean(match); + } + + private boolean isStringMatch(String stringValue) + { + IPAddressString addressString = IPv4AddressExprUtils.parseString(stringValue); + return addressString != null && blockString.prefixContains(addressString); + } + + private boolean isLongMatch(long longValue) + { + IPv4Address address = IPv4AddressExprUtils.parse(longValue); + return address != null && block.contains(address); + } + + @Override + public Expr visit(Shuttle shuttle) + { + return shuttle.visit(apply(shuttle.visitAll(args))); + } + + @Nullable + @Override + public ExpressionType getOutputType(InputBindingInspector inspector) + { + return ExpressionType.LONG; + } + + @Override + public String stringify() + { + return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(ARG_SUBNET).stringify()); } - return ExprEval.ofLongBoolean(match); } - private boolean isStringMatch(String stringValue) - { - IPv4Address iPv4Address = IPv4AddressExprUtils.parse(stringValue); - return iPv4Address != null && subnetString.contains(iPv4Address.toAddressString()); - } - - private boolean isLongMatch(long longValue) - { - IPv4Address iPv4Address = IPv4AddressExprUtils.parse(longValue); - return iPv4Address != null && subnetString.contains(iPv4Address.toAddressString()); - } - - @Override - public Expr visit(Shuttle shuttle) - { - return shuttle.visit(apply(shuttle.visitAll(args))); - } - - @Nullable - @Override - public ExpressionType getOutputType(InputBindingInspector inspector) - { - return ExpressionType.LONG; - } - - @Override - public String stringify() - { - return StringUtils.format("%s(%s, %s)", FN_NAME, arg.stringify(), args.get(ARG_SUBNET).stringify()); - } + return new IPv4AddressMatchExpr(arg); } - return new IPv4AddressMatchExpr(arg, subnetInfo); + catch (AddressStringException e) { + throw new RuntimeException(e); + } } private IPAddressString getSubnetInfo(List args) diff --git a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacroTest.java b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacroTest.java index d62d248f021..54bb71b2988 100644 --- a/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacroTest.java +++ b/processing/src/test/java/org/apache/druid/query/expression/IPv4AddressMatchExprMacroTest.java @@ -177,6 +177,32 @@ public class IPv4AddressMatchExprMacroTest extends MacroTestBase Assert.assertTrue(eval(IPV4_BROADCAST, subnet)); } + @Test + public void testMatchesPrefix() + { + Assert.assertTrue(eval(ExprEval.of("192.168.1.250").toExpr(), ExprEval.of("192.168.1.251/31").toExpr())); + Assert.assertFalse(eval(ExprEval.of("192.168.1.240").toExpr(), ExprEval.of("192.168.1.251/31").toExpr())); + Assert.assertFalse(eval(ExprEval.of("192.168.1.250").toExpr(), ExprEval.of("192.168.1.251/32").toExpr())); + Assert.assertTrue(eval(ExprEval.of("192.168.1.251").toExpr(), ExprEval.of("192.168.1.251/32").toExpr())); + + Assert.assertTrue(eval( + ExprEval.of(IPv4AddressExprUtils.parse("192.168.1.250").longValue()).toExpr(), + ExprEval.of("192.168.1.251/31").toExpr() + )); + Assert.assertFalse(eval( + ExprEval.of(IPv4AddressExprUtils.parse("192.168.1.240").longValue()).toExpr(), + ExprEval.of("192.168.1.251/31").toExpr() + )); + Assert.assertFalse(eval( + ExprEval.of(IPv4AddressExprUtils.parse("192.168.1.250").longValue()).toExpr(), + ExprEval.of("192.168.1.251/32").toExpr() + )); + Assert.assertTrue(eval( + ExprEval.of(IPv4AddressExprUtils.parse("192.168.1.251").longValue()).toExpr(), + ExprEval.of("192.168.1.251/32").toExpr() + )); + } + private boolean eval(Expr... args) { Expr expr = apply(Arrays.asList(args));