SQL: Introduce support for IP fields (#34758)
IP fields are recognized and can be used through-out the query Close #32499
This commit is contained in:
Costin Leau
GitHub
parent
98cd7ca861
commit
7570d69254
|
|
@ -30,7 +30,6 @@ import java.util.Locale;
|
|||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static java.lang.String.format;
|
||||
import static java.util.Calendar.DAY_OF_MONTH;
|
||||
|
|
@ -41,6 +40,7 @@ import static java.util.Calendar.MINUTE;
|
|||
import static java.util.Calendar.MONTH;
|
||||
import static java.util.Calendar.SECOND;
|
||||
import static java.util.Calendar.YEAR;
|
||||
import static java.util.stream.Collectors.toMap;
|
||||
|
||||
/**
|
||||
* Conversion utilities for conversion of JDBC types to Java type and back
|
||||
|
|
@ -52,9 +52,7 @@ import static java.util.Calendar.YEAR;
|
|||
*/
|
||||
final class TypeConverter {
|
||||
|
||||
private TypeConverter() {
|
||||
|
||||
}
|
||||
private TypeConverter() {}
|
||||
|
||||
private static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000;
|
||||
private static final Map<Class<?>, SQLType> javaToJDBC;
|
||||
|
|
@ -64,9 +62,10 @@ final class TypeConverter {
|
|||
Map<Class<?>, SQLType> aMap = Arrays.stream(DataType.values())
|
||||
.filter(dataType -> dataType.javaClass() != null
|
||||
&& dataType != DataType.HALF_FLOAT
|
||||
&& dataType != DataType.IP
|
||||
&& dataType != DataType.SCALED_FLOAT
|
||||
&& dataType != DataType.TEXT)
|
||||
.collect(Collectors.toMap(dataType -> dataType.javaClass(), dataType -> dataType.jdbcType));
|
||||
.collect(toMap(dataType -> dataType.javaClass(), dataType -> dataType.jdbcType));
|
||||
// apart from the mappings in {@code DataType} three more Java classes can be mapped to a {@code JDBCType.TIMESTAMP}
|
||||
// according to B-4 table from the jdbc4.2 spec
|
||||
aMap.put(Calendar.class, JDBCType.TIMESTAMP);
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ import java.util.Arrays;
|
|||
import java.util.HashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static java.util.stream.Collectors.toMap;
|
||||
|
||||
/**
|
||||
* Elasticsearch data types that supported by SQL interface
|
||||
|
|
@ -42,7 +43,13 @@ public enum DataType {
|
|||
// since ODBC and JDBC interpret precision for Date as display size,
|
||||
// the precision is 23 (number of chars in ISO8601 with millis) + Z (the UTC timezone)
|
||||
// see https://github.com/elastic/elasticsearch/issues/30386#issuecomment-386807288
|
||||
DATE( JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES, 24, 24);
|
||||
DATE( JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES, 24, 24),
|
||||
//
|
||||
// specialized types
|
||||
//
|
||||
// IP can be v4 or v6. The latter has 2^128 addresses or 340,282,366,920,938,463,463,374,607,431,768,211,456
|
||||
// aka 39 chars
|
||||
IP( JDBCType.VARCHAR, String.class, 39, 39, 0,false, false, true);
|
||||
// @formatter:on
|
||||
|
||||
public static final String ODBC_DATATYPE_PREFIX = "SQL_";
|
||||
|
|
@ -52,8 +59,9 @@ public enum DataType {
|
|||
|
||||
static {
|
||||
jdbcToEs = Arrays.stream(DataType.values())
|
||||
.filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT) // Remove duplicates
|
||||
.collect(Collectors.toMap(dataType -> dataType.jdbcType, dataType -> dataType));
|
||||
.filter(type -> type != TEXT && type != NESTED
|
||||
&& type != SCALED_FLOAT && type != IP) // Remove duplicates
|
||||
.collect(toMap(dataType -> dataType.jdbcType, dataType -> dataType));
|
||||
|
||||
odbcToEs = new HashMap<>(36);
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
package org.elasticsearch.xpack.sql.expression;
|
||||
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
|
||||
import org.elasticsearch.xpack.sql.expression.Expression.TypeResolution;
|
||||
import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe;
|
||||
|
|
@ -131,16 +132,17 @@ public final class Expressions {
|
|||
}
|
||||
|
||||
public static TypeResolution typeMustBeNumeric(Expression e) {
|
||||
return e.dataType().isNumeric() ? TypeResolution.TYPE_RESOLVED : new TypeResolution(numericErrorMessage(e));
|
||||
return e.dataType().isNumeric() ? TypeResolution.TYPE_RESOLVED : new TypeResolution(incorrectTypeErrorMessage(e, "numeric"));
|
||||
}
|
||||
|
||||
public static TypeResolution typeMustBeNumericOrDate(Expression e) {
|
||||
return e.dataType().isNumeric() || e.dataType() == DataType.DATE ?
|
||||
TypeResolution.TYPE_RESOLVED :
|
||||
new TypeResolution(numericErrorMessage(e));
|
||||
new TypeResolution(incorrectTypeErrorMessage(e, "numeric", "date"));
|
||||
}
|
||||
|
||||
private static String numericErrorMessage(Expression e) {
|
||||
return "Argument required to be numeric ('" + Expressions.name(e) + "' of type '" + e.dataType().esType + "')";
|
||||
private static String incorrectTypeErrorMessage(Expression e, String...acceptedTypes) {
|
||||
return "Argument required to be " + Strings.arrayToDelimitedString(acceptedTypes, " or ")
|
||||
+ " ('" + Expressions.name(e) + "' type is '" + e.dataType().esType + "')";
|
||||
}
|
||||
}
|
||||
|
|
@ -42,7 +42,7 @@ public class In extends NamedExpression implements ScriptWeaver {
|
|||
public In(Location location, Expression value, List<Expression> list) {
|
||||
super(location, null, CollectionUtils.combine(list, value), null);
|
||||
this.value = value;
|
||||
this.list = list.stream().distinct().collect(Collectors.toList());
|
||||
this.list = new ArrayList<>(new LinkedHashSet<>(list));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -67,8 +67,8 @@ public class SysTypes extends Command {
|
|||
@Override
|
||||
public final void execute(SqlSession session, ActionListener<SchemaRowSet> listener) {
|
||||
List<List<?>> rows = Stream.of(DataType.values())
|
||||
// sort by SQL int type (that's what the JDBC/ODBC specs want)
|
||||
.sorted(Comparator.comparing(t -> t.jdbcType.getVendorTypeNumber()))
|
||||
// sort by SQL int type (that's what the JDBC/ODBC specs want) followed by name
|
||||
.sorted(Comparator.comparing((DataType t) -> t.jdbcType.getVendorTypeNumber()).thenComparing(DataType::sqlName))
|
||||
.map(t -> asList(t.esType.toUpperCase(Locale.ROOT),
|
||||
t.jdbcType.getVendorTypeNumber(),
|
||||
//https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/column-size?view=sql-server-2017
|
||||
|
|
|
|||
|
|
@ -139,7 +139,7 @@ public class VerifierErrorMessagesTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testNotSupportedAggregateOnDate() {
|
||||
assertEquals("1:8: Argument required to be numeric ('date' of type 'date')",
|
||||
assertEquals("1:8: Argument required to be numeric ('date' type is 'date')",
|
||||
verify("SELECT AVG(date) FROM test"));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,11 +58,11 @@ public class SysParserTests extends ESTestCase {
|
|||
Command cmd = sql("SYS TYPES").v1();
|
||||
|
||||
List<String> names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "SCALED_FLOAT", "FLOAT", "DOUBLE",
|
||||
"KEYWORD", "TEXT", "BOOLEAN", "DATE", "UNSUPPORTED", "OBJECT", "NESTED");
|
||||
"KEYWORD", "TEXT", "IP", "BOOLEAN", "DATE", "UNSUPPORTED", "OBJECT", "NESTED");
|
||||
|
||||
cmd.execute(null, ActionListener.wrap(r -> {
|
||||
assertEquals(19, r.columnCount());
|
||||
assertEquals(17, r.size());
|
||||
assertEquals(DataType.values().length, r.size());
|
||||
assertFalse(r.schema().types().contains(DataType.NULL));
|
||||
// test numeric as signed
|
||||
assertFalse(r.column(9, Boolean.class));
|
||||
|
|
|
|||
|
|
@ -183,6 +183,13 @@ public class TypesTests extends ESTestCase {
|
|||
assertThat(dt.getDataType().esType, is("unsupported"));
|
||||
}
|
||||
|
||||
public void testIpField() {
|
||||
Map<String, EsField> mapping = loadMapping("mapping-ip.json");
|
||||
assertThat(mapping.size(), is(1));
|
||||
EsField dt = mapping.get("ip_addr");
|
||||
assertThat(dt.getDataType().esType, is("ip"));
|
||||
}
|
||||
|
||||
public void testUnsupportedTypes() {
|
||||
Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
|
||||
EsField dt = mapping.get("range");
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"properties" : {
|
||||
"ip_addr" : {
|
||||
"type" : "ip"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@ package org.elasticsearch.xpack.qa.sql.jdbc;
|
|||
import org.apache.http.HttpHost;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.elasticsearch.client.Request;
|
||||
import org.elasticsearch.client.Response;
|
||||
import org.elasticsearch.client.RestClient;
|
||||
import org.elasticsearch.common.CheckedBiConsumer;
|
||||
import org.elasticsearch.common.Strings;
|
||||
|
|
@ -44,6 +45,7 @@ public class DataLoader {
|
|||
protected static void loadEmpDatasetIntoEs(RestClient client) throws Exception {
|
||||
loadEmpDatasetIntoEs(client, "test_emp", "employees");
|
||||
loadEmpDatasetWithExtraIntoEs(client, "test_emp_copy", "employees");
|
||||
loadLogsDatasetIntoEs(client, "logs", "logs");
|
||||
makeAlias(client, "test_alias", "test_emp", "test_emp_copy");
|
||||
makeAlias(client, "test_alias_emp", "test_emp", "test_emp_copy");
|
||||
}
|
||||
|
|
@ -150,7 +152,7 @@ public class DataLoader {
|
|||
list.add(dep);
|
||||
});
|
||||
|
||||
request = new Request("POST", "/" + index + "/emp/_bulk");
|
||||
request = new Request("POST", "/" + index + "/emp/_bulk?refresh=wait_for");
|
||||
request.addParameter("refresh", "true");
|
||||
StringBuilder bulk = new StringBuilder();
|
||||
csvToLines(fileName, (titles, fields) -> {
|
||||
|
|
@ -193,6 +195,58 @@ public class DataLoader {
|
|||
client.performRequest(request);
|
||||
}
|
||||
|
||||
protected static void loadLogsDatasetIntoEs(RestClient client, String index, String filename) throws Exception {
|
||||
Request request = new Request("PUT", "/" + index);
|
||||
XContentBuilder createIndex = JsonXContent.contentBuilder().startObject();
|
||||
createIndex.startObject("settings");
|
||||
{
|
||||
createIndex.field("number_of_shards", 1);
|
||||
createIndex.field("number_of_replicas", 1);
|
||||
}
|
||||
createIndex.endObject();
|
||||
createIndex.startObject("mappings");
|
||||
{
|
||||
createIndex.startObject("_doc");
|
||||
{
|
||||
createIndex.startObject("properties");
|
||||
{
|
||||
createIndex.startObject("id").field("type", "integer").endObject();
|
||||
createIndex.startObject("@timestamp").field("type", "date").endObject();
|
||||
createIndex.startObject("bytes_in").field("type", "integer").endObject();
|
||||
createIndex.startObject("bytes_out").field("type", "integer").endObject();
|
||||
createIndex.startObject("client_ip").field("type", "ip").endObject();
|
||||
createIndex.startObject("client_port").field("type", "integer").endObject();
|
||||
createIndex.startObject("dest_ip").field("type", "ip").endObject();
|
||||
createIndex.startObject("status").field("type", "keyword").endObject();
|
||||
}
|
||||
createIndex.endObject();
|
||||
}
|
||||
createIndex.endObject();
|
||||
}
|
||||
createIndex.endObject().endObject();
|
||||
request.setJsonEntity(Strings.toString(createIndex));
|
||||
client.performRequest(request);
|
||||
|
||||
request = new Request("POST", "/" + index + "/_doc/_bulk?refresh=wait_for");
|
||||
request.addParameter("refresh", "true");
|
||||
StringBuilder bulk = new StringBuilder();
|
||||
csvToLines(filename, (titles, fields) -> {
|
||||
bulk.append("{\"index\":{\"_id\":\"" + fields.get(0) + "\"}}\n");
|
||||
bulk.append("{");
|
||||
for (int f = 0; f < titles.size(); f++) {
|
||||
if (Strings.hasText(fields.get(f))) {
|
||||
if (f > 0) {
|
||||
bulk.append(",");
|
||||
}
|
||||
bulk.append('"').append(titles.get(f)).append("\":\"").append(fields.get(f)).append('"');
|
||||
}
|
||||
}
|
||||
bulk.append("}\n");
|
||||
});
|
||||
request.setJsonEntity(bulk.toString());
|
||||
Response response = client.performRequest(request);
|
||||
}
|
||||
|
||||
protected static void loadLibDatasetIntoEs(RestClient client, String index) throws Exception {
|
||||
Request request = new Request("PUT", "/" + index);
|
||||
XContentBuilder createIndex = JsonXContent.contentBuilder().startObject();
|
||||
|
|
@ -221,7 +275,7 @@ public class DataLoader {
|
|||
request.setJsonEntity(Strings.toString(createIndex));
|
||||
client.performRequest(request);
|
||||
|
||||
request = new Request("POST", "/" + index + "/book/_bulk");
|
||||
request = new Request("POST", "/" + index + "/book/_bulk?refresh=wait_for");
|
||||
request.addParameter("refresh", "true");
|
||||
StringBuilder bulk = new StringBuilder();
|
||||
csvToLines("library", (titles, fields) -> {
|
||||
|
|
@ -236,7 +290,7 @@ public class DataLoader {
|
|||
bulk.append("}\n");
|
||||
});
|
||||
request.setJsonEntity(bulk.toString());
|
||||
client.performRequest(request);
|
||||
Response response = client.performRequest(request);
|
||||
}
|
||||
|
||||
protected static void makeAlias(RestClient client, String aliasName, String... indices) throws Exception {
|
||||
|
|
|
|||
|
|
@ -152,6 +152,7 @@ showTables
|
|||
SHOW TABLES;
|
||||
|
||||
name | type
|
||||
logs |BASE TABLE
|
||||
test_alias |ALIAS
|
||||
test_alias_emp |ALIAS
|
||||
test_emp |BASE TABLE
|
||||
|
|
|
|||
|
|
@ -0,0 +1,198 @@
|
|||
//
|
||||
// Tests for IP fields
|
||||
//
|
||||
|
||||
selectAll
|
||||
SELECT * FROM logs ORDER BY id LIMIT 10;
|
||||
|
||||
@timestamp | bytes_in | bytes_out | client_ip | client_port | dest_ip | id | status
|
||||
------------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------
|
||||
2017-11-10T21:15:54Z|47 |388 |10.0.1.1 |9152 |172.27.1.129 |1 |OK
|
||||
2017-11-10T21:15:39Z|29 |374 |10.0.1.1 |31693 |172.27.1.123 |2 |OK
|
||||
2017-11-10T21:15:39Z|35 |303 |10.0.1.1 |23625 |172.27.1.113 |3 |OK
|
||||
2017-11-10T21:15:39Z|36 |312 |10.0.1.1 |9932 |172.27.1.116 |4 |OK
|
||||
2017-11-10T21:15:40Z|35 |344 |10.0.1.1 |22695 |172.27.1.149 |5 |OK
|
||||
2017-11-10T21:15:40Z|31 |503 |10.0.1.1 |59811 |172.27.1.122 |6 |OK
|
||||
2017-11-10T21:15:40Z|35 |458 |10.0.1.7 |57372 |172.27.1.140 |7 |OK
|
||||
2017-11-10T21:15:41Z|35 |281 |10.0.1.8 |17370 |null |8 |OK
|
||||
2017-11-10T21:15:41Z|46 |231 |10.0.1.9 |65004 |null |9 |OK
|
||||
2017-11-10T20:36:07Z|40 |506 |10.0.1.10 |22661 |null |10 |OK
|
||||
;
|
||||
|
||||
selectIpField
|
||||
SELECT client_ip, dest_ip FROM logs ORDER BY id LIMIT 10;
|
||||
|
||||
client_ip | dest_ip
|
||||
---------------+---------------
|
||||
10.0.1.1 |172.27.1.129
|
||||
10.0.1.1 |172.27.1.123
|
||||
10.0.1.1 |172.27.1.113
|
||||
10.0.1.1 |172.27.1.116
|
||||
10.0.1.1 |172.27.1.149
|
||||
10.0.1.1 |172.27.1.122
|
||||
10.0.1.7 |172.27.1.140
|
||||
10.0.1.8 |null
|
||||
10.0.1.9 |null
|
||||
10.0.1.10 |null
|
||||
;
|
||||
|
||||
orderByIpv4Field
|
||||
SELECT client_ip, dest_ip FROM logs ORDER BY client_ip LIMIT 5;
|
||||
|
||||
client_ip | dest_ip
|
||||
---------------+------------------------------
|
||||
10.0.0.105 |172.27.1.1
|
||||
10.0.0.107 |172.20.10.8
|
||||
10.0.0.109 |2001:cafe::470f:60b7:f84a:25b6
|
||||
10.0.0.113 |90.128.199.24
|
||||
10.0.0.118 |172.27.1.1
|
||||
;
|
||||
|
||||
orderByIpv6Field
|
||||
SELECT client_ip, dest_ip FROM logs ORDER BY dest_ip ASC LIMIT 5;
|
||||
|
||||
client_ip:s | dest_ip:s
|
||||
---------------+------------------------------
|
||||
null |27.58.6.220
|
||||
10.0.0.147 |90.128.199.24
|
||||
10.0.0.113 |90.128.199.24
|
||||
10.0.0.129 |172.16.1.1
|
||||
10.0.1.177 |172.20.10.1
|
||||
;
|
||||
|
||||
filterExactMatchIpv4
|
||||
SELECT id, client_ip, dest_ip FROM logs WHERE client_ip = '10.0.1.166' ORDER BY id LIMIT 5;
|
||||
|
||||
id | client_ip | dest_ip
|
||||
---------------+---------------+------------------------------
|
||||
22 |10.0.1.166 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
24 |10.0.1.166 |2001:cafe::13e1:16fc:8726:1bf8
|
||||
29 |10.0.1.166 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
33 |10.0.1.166 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
34 |10.0.1.166 |2001:cafe::ff07:bdcc:bc59:ff9e
|
||||
;
|
||||
|
||||
filterExactMatchIpv6
|
||||
SELECT id, client_ip, dest_ip FROM logs WHERE dest_ip = 'fe80::86ba:3bff:fe05:c3f3' ORDER BY id LIMIT 10;
|
||||
|
||||
id | client_ip | dest_ip
|
||||
---------------+---------------+-------------------------
|
||||
19 |10.0.1.13 |fe80::86ba:3bff:fe05:c3f3
|
||||
;
|
||||
|
||||
|
||||
filterRangeIpv4
|
||||
SELECT id, client_ip, dest_ip FROM logs WHERE client_ip BETWEEN '10.0.1.1' AND '10.0.1.200' ORDER BY id LIMIT 10;
|
||||
|
||||
id | client_ip | dest_ip
|
||||
---------------+---------------+---------------
|
||||
1 |10.0.1.1 |172.27.1.129
|
||||
2 |10.0.1.1 |172.27.1.123
|
||||
3 |10.0.1.1 |172.27.1.113
|
||||
4 |10.0.1.1 |172.27.1.116
|
||||
5 |10.0.1.1 |172.27.1.149
|
||||
6 |10.0.1.1 |172.27.1.122
|
||||
7 |10.0.1.7 |172.27.1.140
|
||||
8 |10.0.1.8 |null
|
||||
9 |10.0.1.9 |null
|
||||
10 |10.0.1.10 |null
|
||||
;
|
||||
|
||||
filterRangeCIDRIpv4
|
||||
SELECT id, client_ip, dest_ip FROM logs WHERE client_ip = '10.0.0.0/16' ORDER BY id LIMIT 5;
|
||||
|
||||
id | client_ip | dest_ip
|
||||
---------------+---------------+---------------
|
||||
1 |10.0.1.1 |172.27.1.129
|
||||
2 |10.0.1.1 |172.27.1.123
|
||||
3 |10.0.1.1 |172.27.1.113
|
||||
4 |10.0.1.1 |172.27.1.116
|
||||
5 |10.0.1.1 |172.27.1.149
|
||||
;
|
||||
|
||||
filterRangeCIDRIpv6
|
||||
SELECT id, client_ip, dest_ip FROM logs WHERE dest_ip = '2001:cafe::/48' ORDER BY id LIMIT 5;
|
||||
|
||||
id | client_ip | dest_ip
|
||||
---------------+---------------+------------------------------
|
||||
20 |10.0.1.199 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
22 |10.0.1.166 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
23 |null |2001:cafe::d46a:9bdc:8126:b00b
|
||||
24 |10.0.1.166 |2001:cafe::13e1:16fc:8726:1bf8
|
||||
25 |10.0.1.199 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
;
|
||||
|
||||
//
|
||||
// waiting on https://github.com/elastic/elasticsearch/issues/34799
|
||||
//
|
||||
filterInCIDRIpv4-Ignore
|
||||
SELECT id, client_ip, dest_ip FROM logs WHERE dest_ip IN ('10.0.1.1', '10.0.1.200', '10.0.0.0/16') ORDER BY id LIMIT 10;
|
||||
|
||||
id | client_ip | dest_ip
|
||||
---------------+---------------+------------------------------
|
||||
20 |10.0.1.199 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
22 |10.0.1.166 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
23 |10.0.1.199 |2001:cafe::d46a:9bdc:8126:b00b
|
||||
24 |10.0.1.166 |2001:cafe::13e1:16fc:8726:1bf8
|
||||
;
|
||||
|
||||
|
||||
filterInCIDRIpv6-Ignore
|
||||
SELECT id, client_ip, dest_ip FROM logs WHERE dest_ip IN ('127.0.0.1', '2001:cafe::13e1:16fc:8726:1bf8', '2001:cafe::/48') ORDER BY id LIMIT 10;
|
||||
|
||||
id | client_ip | dest_ip
|
||||
---------------+---------------+------------------------------
|
||||
20 |10.0.1.199 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
22 |10.0.1.166 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
23 |10.0.1.199 |2001:cafe::d46a:9bdc:8126:b00b
|
||||
24 |10.0.1.166 |2001:cafe::13e1:16fc:8726:1bf8
|
||||
;
|
||||
|
||||
groupByIpv4
|
||||
SELECT client_ip FROM logs GROUP BY client_ip LIMIT 5;
|
||||
|
||||
client_ip:s
|
||||
---------------
|
||||
null
|
||||
10.0.0.105
|
||||
10.0.0.107
|
||||
10.0.0.109
|
||||
10.0.0.113
|
||||
;
|
||||
|
||||
groupByIpv6
|
||||
SELECT dest_ip FROM logs GROUP BY dest_ip ORDER BY dest_ip DESC LIMIT 5;
|
||||
|
||||
dest_ip
|
||||
------------------------------
|
||||
fe80::a65e:60ff:fee8:fee9
|
||||
fe80::86ba:3bff:fe05:c3f3
|
||||
2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
2001:cafe::ff07:bdcc:bc59:ff9e
|
||||
2001:cafe::ff07:bdcc:bc59:ff9d
|
||||
;
|
||||
|
||||
groupByIpv4AndIpv6
|
||||
SELECT client_ip, dest_ip FROM logs GROUP BY client_ip, dest_ip ORDER BY dest_ip DESC LIMIT 5;
|
||||
|
||||
client_ip | dest_ip
|
||||
---------------+------------------------------
|
||||
10.0.1.222 |fe80::a65e:60ff:fee8:fee9
|
||||
10.0.1.13 |fe80::86ba:3bff:fe05:c3f3
|
||||
null |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
10.0.1.166 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
10.0.1.199 |2001:cafe::ff07:bdcc:bc59:ff9f
|
||||
;
|
||||
|
||||
|
||||
groupByIpv4AndPort
|
||||
SELECT client_ip, client_port FROM logs GROUP BY client_ip, client_port ORDER BY client_port DESC LIMIT 5;
|
||||
|
||||
client_ip | client_port
|
||||
---------------+---------------
|
||||
10.0.1.9 |65004
|
||||
10.0.0.129 |63982
|
||||
null |63238
|
||||
null |61337
|
||||
null |61220
|
||||
;
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
id,@timestamp,bytes_in,bytes_out,client_ip,client_port,dest_ip,status
|
||||
1,2017-11-10T21:15:54Z,47,388,10.0.1.1,9152,172.27.1.129,OK
|
||||
2,2017-11-10T21:15:39Z,29,374,10.0.1.1,31693,172.27.1.123,OK
|
||||
3,2017-11-10T21:15:39Z,35,303,10.0.1.1,23625,172.27.1.113,OK
|
||||
4,2017-11-10T21:15:39Z,36,312,10.0.1.1,9932,172.27.1.116,OK
|
||||
5,2017-11-10T21:15:40Z,35,344,10.0.1.1,22695,172.27.1.149,OK
|
||||
6,2017-11-10T21:15:40Z,31,503,10.0.1.1,59811,172.27.1.122,OK
|
||||
7,2017-11-10T21:15:40Z,35,458,10.0.1.7,57372,172.27.1.140,OK
|
||||
8,2017-11-10T21:15:41Z,35,281,10.0.1.8,17370,,OK
|
||||
9,2017-11-10T21:15:41Z,46,231,10.0.1.9,65004,,OK
|
||||
10,2017-11-10T20:36:07Z,40,506,10.0.1.10,22661,,OK
|
||||
11,2017-11-10T20:36:08Z,34,471,10.0.1.11,16752,172.27.1.131,OK
|
||||
12,2017-11-10T20:36:07Z,39,503,10.0.1.12,19479,172.27.1.103,OK
|
||||
13,2017-11-10T20:36:07Z,29,502,10.0.1.13,2326,172.27.1.139,OK
|
||||
14,2017-11-10T20:36:15Z,35,280,10.0.1.13,51758,172.27.1.129,OK
|
||||
15,2017-11-10T20:36:15Z,38,225,,22994,172.27.1.139,OK
|
||||
16,2017-11-10T20:35:54Z,35,326,,5505,172.27.1.120,OK
|
||||
17,2017-11-10T20:35:54Z,46,466,10.0.1.13,3666,172.27.1.103,OK
|
||||
18,2017-11-10T20:35:55Z,42,238,10.0.1.13,23791,172.27.1.111,OK
|
||||
19,2017-11-10T17:54:43Z,16,,10.0.1.13,,fe80::86ba:3bff:fe05:c3f3,OK
|
||||
20,2017-11-10T23:23:24Z,40,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
21,2017-11-10T17:54:59Z,24,,10.0.1.222,,fe80::a65e:60ff:fee8:fee9,OK
|
||||
22,2017-11-10T21:13:27Z,20,,10.0.1.166,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
23,2017-11-10T22:37:41Z,24,,,,2001:cafe::d46a:9bdc:8126:b00b,OK
|
||||
24,2017-11-10T20:34:43Z,8,,10.0.1.166,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
25,2017-11-10T23:30:46Z,40,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
26,2017-11-10T21:13:16Z,20,,,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
27,2017-11-10T23:36:32Z,0,,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
28,2017-11-10T23:36:33Z,40,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
29,2017-11-10T20:35:26Z,20,,10.0.1.166,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
30,2017-11-10T23:36:41Z,8,,,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
31,2017-11-10T23:56:36Z,8,,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
32,2017-11-10T20:29:25Z,32,,10.0.1.177,59769,172.20.10.1,Error
|
||||
33,2017-11-10T21:35:01Z,20,,10.0.1.166,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
34,2017-11-10T21:12:17Z,20,,10.0.1.166,,2001:cafe::ff07:bdcc:bc59:ff9e,OK
|
||||
35,2017-11-10T23:17:14Z,40,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9d,OK
|
||||
36,2017-11-10T23:28:11Z,8,,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
37,2017-11-10T22:36:27Z,8,,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
38,2017-11-10T20:35:55Z,36,281,,58533,172.27.1.1,OK
|
||||
39,2017-11-10T20:35:55Z,25,273,,39211,,OK
|
||||
40,2017-11-10T20:35:55Z,34,253,,37971,172.27.1.1,OK
|
||||
41,2017-11-10T20:35:55Z,41,503,,47831,172.27.1.1,OK
|
||||
42,2017-11-10T21:34:49Z,28,,,,27.58.6.220,Error
|
||||
43,2017-11-10T20:35:55Z,28,206,10.0.1.200,31000,172.27.1.1,OK
|
||||
44,2017-11-10T20:14:04Z,8,,10.0.1.201,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
45,2017-11-10T19:38:06Z,37,239,10.0.1.202,3577,172.27.1.1,OK
|
||||
46,2017-11-10T21:14:18Z,8,,10.0.1.203,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
47,2017-11-10T20:35:56Z,34,202,10.0.1.204,49112,172.27.1.1,OK
|
||||
48,2017-11-10T20:53:05Z,8,,10.0.1.205,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
49,2017-11-10T21:25:42Z,8,,10.0.1.206,,2001:cafe::13e1:16fc:8726:1bf9,OK
|
||||
50,2017-11-10T21:14:44Z,8,,10.0.1.207,,2001:cafe::13e1:16fc:8726:1bf4,OK
|
||||
51,2017-11-10T21:28:34Z,8,,10.0.1.208,,2001:cafe::13e1:16fc:8726:1bf3,OK
|
||||
52,2017-11-10T20:35:55Z,34,227,,63238,172.27.1.1,OK
|
||||
53,2017-11-10T20:15:24Z,8,,,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
54,2017-11-10T20:35:57Z,37,239,,61337,172.27.1.1,OK
|
||||
55,2017-11-10T17:14:10Z,16,,10.0.1.222,,2001:cafe::a98d:374:79e4:4865,OK
|
||||
56,2017-11-10T20:35:57Z,38,476,10.0.1.200,53720,172.27.1.1,OK
|
||||
57,2017-11-10T23:22:13Z,8,,10.0.1.201,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
58,2017-11-10T20:32:57Z,8,,10.0.1.202,,2001:cafe::13e1:16fc:8726:1bf7,OK
|
||||
59,2017-11-10T21:24:00Z,8,,10.0.1.203,,2001:cafe::13e1:16fc:8726:1bf6,OK
|
||||
60,2017-11-10T20:35:56Z,32,503,10.0.1.204,19382,172.27.1.1,OK
|
||||
61,2017-11-10T23:43:10Z,0,,10.0.1.205,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
62,2017-11-10T20:35:57Z,30,169,10.0.1.206,47532,172.27.1.1,OK
|
||||
63,2017-11-10T20:21:58Z,20,,10.0.1.207,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
64,2017-11-10T20:35:57Z,41,271,10.0.1.208,16227,,OK
|
||||
65,2017-11-10T20:33:06Z,28,,10.0.1.166,,172.27.1.1,Error
|
||||
66,2017-11-10T20:35:57Z,33,185,,28928,172.27.1.1,OK
|
||||
67,2017-11-10T20:26:21Z,20,,,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
68,2017-11-10T21:23:25Z,20,,,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
69,2017-11-10T21:23:54Z,8,,10.0.1.166,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
70,2017-11-10T20:35:57Z,35,234,10.0.1.166,54994,172.27.1.1,OK
|
||||
71,2017-11-10T00:27:03Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b6,OK
|
||||
72,2017-11-10T00:27:46Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b6,OK
|
||||
73,2017-11-10T20:35:58Z,35,223,,20163,172.27.1.1,OK
|
||||
74,2017-11-10T20:35:57Z,32,501,10.0.1.166,51275,172.27.1.1,OK
|
||||
75,2017-11-10T22:27:09Z,20,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
76,2017-11-10T20:35:58Z,45,493,10.0.1.166,1193,172.27.1.1,OK
|
||||
77,2017-11-10T22:26:44Z,20,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
|
||||
78,2017-11-10T22:27:31Z,8,,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK
|
||||
79,2017-11-10T20:35:52Z,47,246,,24564,172.27.1.1,OK
|
||||
80,2017-11-10T00:00:22Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b6,OK
|
||||
81,2017-11-10T20:35:52Z,37,420,10.0.1.166,40542,172.27.1.1,OK
|
||||
82,2017-11-10T00:01:20Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b6,OK
|
||||
83,2017-11-10T00:01:04Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b6,OK
|
||||
84,2017-11-10T00:32:48Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b5,OK
|
||||
85,2017-11-10T00:01:45Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b4,OK
|
||||
86,2017-11-10T20:36:08Z,38,509,,61220,172.27.1.1,OK
|
||||
87,2017-11-10T21:17:37Z,38,226,10.0.0.144,26602,,OK
|
||||
88,2017-11-10T20:06:49Z,30,,10.0.0.147,53240,90.128.199.24,Error
|
||||
89,2017-11-10T21:17:37Z,44,284,10.0.0.118,49479,172.27.1.1,OK
|
||||
90,2017-11-10T19:51:38Z,28,,10.0.0.130,,203.131.98.151,Error
|
||||
91,2017-11-10T19:51:38Z,28,,10.0.0.107,,172.20.10.8,Error
|
||||
92,2017-11-10T20:06:50Z,34,215,10.0.0.113,25162,90.128.199.24,OK
|
||||
93,2017-11-10T21:17:46Z,33,185,10.0.0.129,63982,172.27.1.1,OK
|
||||
94,2017-11-10T19:51:38Z,28,,10.0.0.130,,203.131.98.151,Error
|
||||
95,2017-11-10T21:17:46Z,28,321,10.0.0.105,4292,172.27.1.1,OK
|
||||
96,2017-11-10T00:04:50Z,48,,10.0.0.109,,2001:cafe::470f:60b7:f84a:25b6,OK
|
||||
97,2017-11-10T21:17:48Z,30,280,10.0.0.145,57783,172.27.1.1,OK
|
||||
98,2017-11-10T21:12:24Z,74,90,10.0.0.134,57203,172.20.10.1,OK
|
||||
99,2017-11-10T21:17:37Z,39,512,10.0.0.128,29333,,OK
|
||||
100,2017-11-10T03:21:36Z,64,183,10.0.0.129,4541,172.16.1.1,OK
|
||||
|
Loading…
Reference in New Issue