SQL: Introduce support for IP fields (#34758)

IP fields are recognized and can be used through-out the query

Close #32499

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/jdbc/TypeConverter.java

@@ -30,7 +30,6 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 
 import static java.lang.String.format;
 import static java.util.Calendar.DAY_OF_MONTH;
@@ -41,6 +40,7 @@ import static java.util.Calendar.MINUTE;
 import static java.util.Calendar.MONTH;
 import static java.util.Calendar.SECOND;
 import static java.util.Calendar.YEAR;
+import static java.util.stream.Collectors.toMap;
 
 /**
  * Conversion utilities for conversion of JDBC types to Java type and back
@@ -52,9 +52,7 @@ import static java.util.Calendar.YEAR;
  */
 final class TypeConverter {
 
-    private TypeConverter() {
+    private TypeConverter() {}
-
-    }
 
     private static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000;
     private static final Map<Class<?>, SQLType> javaToJDBC;
@@ -64,9 +62,10 @@ final class TypeConverter {
         Map<Class<?>, SQLType> aMap = Arrays.stream(DataType.values())
                 .filter(dataType -> dataType.javaClass() != null
                         && dataType != DataType.HALF_FLOAT
+                        && dataType != DataType.IP
                         && dataType != DataType.SCALED_FLOAT
                         && dataType != DataType.TEXT)
-                .collect(Collectors.toMap(dataType -> dataType.javaClass(), dataType -> dataType.jdbcType));
+                .collect(toMap(dataType -> dataType.javaClass(), dataType -> dataType.jdbcType));
         // apart from the mappings in {@code DataType} three more Java classes can be mapped to a {@code JDBCType.TIMESTAMP}
         // according to B-4 table from the jdbc4.2 spec
         aMap.put(Calendar.class, JDBCType.TIMESTAMP);

x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java

@@ -12,7 +12,8 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
-import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.toMap;
 
 /**
  * Elasticsearch data types that supported by SQL interface
@@ -42,7 +43,13 @@ public enum DataType {
     // since ODBC and JDBC interpret precision for Date as display size,
     // the precision is 23 (number of chars in ISO8601 with millis) + Z (the UTC timezone)
     // see https://github.com/elastic/elasticsearch/issues/30386#issuecomment-386807288
-    DATE(        JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES,        24,                24);
+    DATE(        JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES,        24,                24),
+    //
+    // specialized types
+    //
+    // IP can be v4 or v6. The latter has 2^128 addresses or 340,282,366,920,938,463,463,374,607,431,768,211,456
+    // aka 39 chars
+    IP(          JDBCType.VARCHAR,   String.class,     39,               39,                0,false, false, true);
     // @formatter:on
 
     public static final String ODBC_DATATYPE_PREFIX = "SQL_";
@@ -52,8 +59,9 @@ public enum DataType {
 
     static {
         jdbcToEs = Arrays.stream(DataType.values())
-                .filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT) // Remove duplicates
+                .filter(type -> type != TEXT && type != NESTED
-                .collect(Collectors.toMap(dataType -> dataType.jdbcType, dataType -> dataType));
+                                && type != SCALED_FLOAT && type != IP) // Remove duplicates
+                .collect(toMap(dataType -> dataType.jdbcType, dataType -> dataType));
 
         odbcToEs = new HashMap<>(36);
 
@@ -238,4 +246,4 @@ public enum DataType {
             (isString() && other.isString()) ||
             (isNumeric() && other.isNumeric());
     }
-}
+}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.sql.expression;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
 import org.elasticsearch.xpack.sql.expression.Expression.TypeResolution;
 import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe;
@@ -131,16 +132,17 @@ public final class Expressions {
     }
 
     public static TypeResolution typeMustBeNumeric(Expression e) {
-        return e.dataType().isNumeric() ? TypeResolution.TYPE_RESOLVED : new TypeResolution(numericErrorMessage(e));
+        return e.dataType().isNumeric() ? TypeResolution.TYPE_RESOLVED : new TypeResolution(incorrectTypeErrorMessage(e, "numeric"));
     }
 
     public static TypeResolution typeMustBeNumericOrDate(Expression e) {
         return e.dataType().isNumeric() || e.dataType() == DataType.DATE ?
             TypeResolution.TYPE_RESOLVED :
-            new TypeResolution(numericErrorMessage(e));
+                new TypeResolution(incorrectTypeErrorMessage(e, "numeric", "date"));
     }
-
+    
-    private static String numericErrorMessage(Expression e) {
+    private static String incorrectTypeErrorMessage(Expression e, String...acceptedTypes) {
-        return "Argument required to be numeric ('" + Expressions.name(e) + "' of type '" + e.dataType().esType + "')";
+        return "Argument required to be " + Strings.arrayToDelimitedString(acceptedTypes, " or ")
+                + " ('" + Expressions.name(e) + "' type is '" + e.dataType().esType + "')";
     }
-}
+}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/In.java

@@ -42,7 +42,7 @@ public class In extends NamedExpression implements ScriptWeaver {
     public In(Location location, Expression value, List<Expression> list) {
         super(location, null, CollectionUtils.combine(list, value), null);
         this.value = value;
-        this.list = list.stream().distinct().collect(Collectors.toList());
+        this.list = new ArrayList<>(new LinkedHashSet<>(list));
     }
 
     @Override

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java

@@ -67,8 +67,8 @@ public class SysTypes extends Command {
     @Override
     public final void execute(SqlSession session, ActionListener<SchemaRowSet> listener) {
         List<List<?>> rows = Stream.of(DataType.values())
-                // sort by SQL int type (that's what the JDBC/ODBC specs want)
+                // sort by SQL int type (that's what the JDBC/ODBC specs want) followed by name
-                .sorted(Comparator.comparing(t -> t.jdbcType.getVendorTypeNumber()))
+                .sorted(Comparator.comparing((DataType t) -> t.jdbcType.getVendorTypeNumber()).thenComparing(DataType::sqlName))
                 .map(t -> asList(t.esType.toUpperCase(Locale.ROOT),
                         t.jdbcType.getVendorTypeNumber(),
                         //https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/column-size?view=sql-server-2017

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java

@@ -139,7 +139,7 @@ public class VerifierErrorMessagesTests extends ESTestCase {
     }
 
     public void testNotSupportedAggregateOnDate() {
-        assertEquals("1:8: Argument required to be numeric ('date' of type 'date')",
+        assertEquals("1:8: Argument required to be numeric ('date' type is 'date')",
             verify("SELECT AVG(date) FROM test"));
     }
 

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java

@@ -58,11 +58,11 @@ public class SysParserTests extends ESTestCase {
         Command cmd = sql("SYS TYPES").v1();
 
         List<String> names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "SCALED_FLOAT", "FLOAT", "DOUBLE",
-                "KEYWORD", "TEXT", "BOOLEAN", "DATE", "UNSUPPORTED", "OBJECT", "NESTED");
+                "KEYWORD", "TEXT", "IP", "BOOLEAN", "DATE", "UNSUPPORTED", "OBJECT", "NESTED");
 
         cmd.execute(null, ActionListener.wrap(r -> {
             assertEquals(19, r.columnCount());
-            assertEquals(17, r.size());
+            assertEquals(DataType.values().length, r.size());
             assertFalse(r.schema().types().contains(DataType.NULL));
             // test numeric as signed
             assertFalse(r.column(9, Boolean.class));

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java

@@ -183,6 +183,13 @@ public class TypesTests extends ESTestCase {
         assertThat(dt.getDataType().esType, is("unsupported"));
     }
 
+    public void testIpField() {
+        Map<String, EsField> mapping = loadMapping("mapping-ip.json");
+        assertThat(mapping.size(), is(1));
+        EsField dt = mapping.get("ip_addr");
+        assertThat(dt.getDataType().esType, is("ip"));
+    }
+
     public void testUnsupportedTypes() {
         Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
         EsField dt = mapping.get("range");

x-pack/plugin/sql/src/test/resources/mapping-ip.json

@@ -0,0 +1,7 @@
+{
+    "properties" : {
+        "ip_addr" : {
+            "type" : "ip"
+        }
+    }
+}

x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.qa.sql.jdbc;
 import org.apache.http.HttpHost;
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.Strings;
@@ -44,6 +45,7 @@ public class DataLoader {
     protected static void loadEmpDatasetIntoEs(RestClient client) throws Exception {
         loadEmpDatasetIntoEs(client, "test_emp", "employees");
         loadEmpDatasetWithExtraIntoEs(client, "test_emp_copy", "employees");
+        loadLogsDatasetIntoEs(client, "logs", "logs");
         makeAlias(client, "test_alias", "test_emp", "test_emp_copy");
         makeAlias(client, "test_alias_emp", "test_emp", "test_emp_copy");
     }
@@ -150,7 +152,7 @@ public class DataLoader {
             list.add(dep);
         });
 
-        request = new Request("POST", "/" + index + "/emp/_bulk");
+        request = new Request("POST", "/" + index + "/emp/_bulk?refresh=wait_for");
         request.addParameter("refresh", "true");
         StringBuilder bulk = new StringBuilder();
         csvToLines(fileName, (titles, fields) -> {
@@ -193,6 +195,58 @@ public class DataLoader {
         client.performRequest(request);
     }
 
+    protected static void loadLogsDatasetIntoEs(RestClient client, String index, String filename) throws Exception {
+        Request request = new Request("PUT", "/" + index);
+        XContentBuilder createIndex = JsonXContent.contentBuilder().startObject();
+        createIndex.startObject("settings");
+        {
+            createIndex.field("number_of_shards", 1);
+            createIndex.field("number_of_replicas", 1);
+        }
+        createIndex.endObject();
+        createIndex.startObject("mappings");
+        {
+            createIndex.startObject("_doc");
+            {
+                createIndex.startObject("properties");
+                {
+                    createIndex.startObject("id").field("type", "integer").endObject();
+                    createIndex.startObject("@timestamp").field("type", "date").endObject();
+                    createIndex.startObject("bytes_in").field("type", "integer").endObject();
+                    createIndex.startObject("bytes_out").field("type", "integer").endObject();
+                    createIndex.startObject("client_ip").field("type", "ip").endObject();
+                    createIndex.startObject("client_port").field("type", "integer").endObject();
+                    createIndex.startObject("dest_ip").field("type", "ip").endObject();
+                    createIndex.startObject("status").field("type", "keyword").endObject();
+                }
+                createIndex.endObject();
+            }
+            createIndex.endObject();
+        }
+        createIndex.endObject().endObject();
+        request.setJsonEntity(Strings.toString(createIndex));
+        client.performRequest(request);
+
+        request = new Request("POST", "/" + index + "/_doc/_bulk?refresh=wait_for");
+        request.addParameter("refresh", "true");
+        StringBuilder bulk = new StringBuilder();
+        csvToLines(filename, (titles, fields) -> {
+            bulk.append("{\"index\":{\"_id\":\"" + fields.get(0) + "\"}}\n");
+            bulk.append("{");
+            for (int f = 0; f < titles.size(); f++) {
+                if (Strings.hasText(fields.get(f))) {
+                    if (f > 0) {
+                        bulk.append(",");
+                    }
+                    bulk.append('"').append(titles.get(f)).append("\":\"").append(fields.get(f)).append('"');
+                }
+            }
+            bulk.append("}\n");
+        });
+        request.setJsonEntity(bulk.toString());
+        Response response = client.performRequest(request);
+    }
+
     protected static void loadLibDatasetIntoEs(RestClient client, String index) throws Exception {
         Request request = new Request("PUT", "/" + index);
         XContentBuilder createIndex = JsonXContent.contentBuilder().startObject();
@@ -221,7 +275,7 @@ public class DataLoader {
         request.setJsonEntity(Strings.toString(createIndex));
         client.performRequest(request);
 
-        request = new Request("POST", "/" + index + "/book/_bulk");
+        request = new Request("POST", "/" + index + "/book/_bulk?refresh=wait_for");
         request.addParameter("refresh", "true");
         StringBuilder bulk = new StringBuilder();
         csvToLines("library", (titles, fields) -> {
@@ -236,7 +290,7 @@ public class DataLoader {
             bulk.append("}\n");
         });
         request.setJsonEntity(bulk.toString());
-        client.performRequest(request);
+        Response response = client.performRequest(request);
     }
 
     protected static void makeAlias(RestClient client, String aliasName, String... indices) throws Exception {
@@ -270,4 +324,4 @@ public class DataLoader {
     public static InputStream readFromJarUrl(URL source) throws IOException {
         return source.openStream();
     }
-}
+}

x-pack/qa/sql/src/main/resources/command.csv-spec

@@ -152,6 +152,7 @@ showTables
 SHOW TABLES;
 
        name        |     type      
+logs               |BASE TABLE
 test_alias         |ALIAS          
 test_alias_emp     |ALIAS          
 test_emp           |BASE TABLE     

x-pack/qa/sql/src/main/resources/ip.csv-spec

@@ -0,0 +1,198 @@
+//
+// Tests for IP fields
+//
+
+selectAll
+SELECT * FROM logs ORDER BY id LIMIT 10;
+
+       @timestamp       |   bytes_in    |   bytes_out   |   client_ip   |  client_port  |    dest_ip    |      id       |    status     
+------------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------
+2017-11-10T21:15:54Z|47             |388            |10.0.1.1       |9152           |172.27.1.129   |1              |OK             
+2017-11-10T21:15:39Z|29             |374            |10.0.1.1       |31693          |172.27.1.123   |2              |OK             
+2017-11-10T21:15:39Z|35             |303            |10.0.1.1       |23625          |172.27.1.113   |3              |OK             
+2017-11-10T21:15:39Z|36             |312            |10.0.1.1       |9932           |172.27.1.116   |4              |OK             
+2017-11-10T21:15:40Z|35             |344            |10.0.1.1       |22695          |172.27.1.149   |5              |OK             
+2017-11-10T21:15:40Z|31             |503            |10.0.1.1       |59811          |172.27.1.122   |6              |OK             
+2017-11-10T21:15:40Z|35             |458            |10.0.1.7       |57372          |172.27.1.140   |7              |OK             
+2017-11-10T21:15:41Z|35             |281            |10.0.1.8       |17370          |null           |8              |OK             
+2017-11-10T21:15:41Z|46             |231            |10.0.1.9       |65004          |null           |9              |OK             
+2017-11-10T20:36:07Z|40             |506            |10.0.1.10      |22661          |null           |10             |OK    
+;
+
+selectIpField
+SELECT client_ip, dest_ip FROM logs ORDER BY id LIMIT 10;
+
+   client_ip   |    dest_ip    
+---------------+---------------
+10.0.1.1       |172.27.1.129   
+10.0.1.1       |172.27.1.123   
+10.0.1.1       |172.27.1.113   
+10.0.1.1       |172.27.1.116   
+10.0.1.1       |172.27.1.149   
+10.0.1.1       |172.27.1.122   
+10.0.1.7       |172.27.1.140   
+10.0.1.8       |null           
+10.0.1.9       |null           
+10.0.1.10      |null   
+;
+
+orderByIpv4Field
+SELECT client_ip, dest_ip FROM logs ORDER BY client_ip LIMIT 5;
+
+   client_ip   |           dest_ip            
+---------------+------------------------------
+10.0.0.105     |172.27.1.1                    
+10.0.0.107     |172.20.10.8                   
+10.0.0.109     |2001:cafe::470f:60b7:f84a:25b6
+10.0.0.113     |90.128.199.24                 
+10.0.0.118     |172.27.1.1      
+;
+
+orderByIpv6Field
+SELECT client_ip, dest_ip FROM logs ORDER BY dest_ip ASC LIMIT 5;
+
+   client_ip:s |           dest_ip:s            
+---------------+------------------------------
+null           |27.58.6.220    
+10.0.0.147     |90.128.199.24  
+10.0.0.113     |90.128.199.24  
+10.0.0.129     |172.16.1.1     
+10.0.1.177     |172.20.10.1 
+;
+
+filterExactMatchIpv4
+SELECT id, client_ip, dest_ip FROM logs WHERE client_ip = '10.0.1.166' ORDER BY id LIMIT 5;
+
+      id       |   client_ip   |           dest_ip            
+---------------+---------------+------------------------------
+22             |10.0.1.166     |2001:cafe::ff07:bdcc:bc59:ff9f
+24             |10.0.1.166     |2001:cafe::13e1:16fc:8726:1bf8
+29             |10.0.1.166     |2001:cafe::ff07:bdcc:bc59:ff9f
+33             |10.0.1.166     |2001:cafe::ff07:bdcc:bc59:ff9f
+34             |10.0.1.166     |2001:cafe::ff07:bdcc:bc59:ff9e
+;
+
+filterExactMatchIpv6
+SELECT id, client_ip, dest_ip FROM logs WHERE dest_ip = 'fe80::86ba:3bff:fe05:c3f3' ORDER BY id LIMIT 10;
+
+      id       |   client_ip   |         dest_ip         
+---------------+---------------+-------------------------
+19             |10.0.1.13      |fe80::86ba:3bff:fe05:c3f3
+;
+
+
+filterRangeIpv4
+SELECT id, client_ip, dest_ip FROM logs WHERE client_ip BETWEEN '10.0.1.1' AND '10.0.1.200' ORDER BY id LIMIT 10;
+
+      id       |   client_ip   |    dest_ip    
+---------------+---------------+---------------
+1              |10.0.1.1       |172.27.1.129   
+2              |10.0.1.1       |172.27.1.123   
+3              |10.0.1.1       |172.27.1.113   
+4              |10.0.1.1       |172.27.1.116   
+5              |10.0.1.1       |172.27.1.149   
+6              |10.0.1.1       |172.27.1.122   
+7              |10.0.1.7       |172.27.1.140   
+8              |10.0.1.8       |null           
+9              |10.0.1.9       |null           
+10             |10.0.1.10      |null    
+;
+
+filterRangeCIDRIpv4
+SELECT id, client_ip, dest_ip FROM logs WHERE client_ip = '10.0.0.0/16' ORDER BY id LIMIT 5;
+
+      id       |   client_ip   |    dest_ip    
+---------------+---------------+---------------
+1              |10.0.1.1       |172.27.1.129   
+2              |10.0.1.1       |172.27.1.123   
+3              |10.0.1.1       |172.27.1.113   
+4              |10.0.1.1       |172.27.1.116   
+5              |10.0.1.1       |172.27.1.149   
+;
+
+filterRangeCIDRIpv6
+SELECT id, client_ip, dest_ip FROM logs WHERE dest_ip = '2001:cafe::/48' ORDER BY id LIMIT 5;
+
+      id       |   client_ip   |           dest_ip            
+---------------+---------------+------------------------------
+20             |10.0.1.199     |2001:cafe::ff07:bdcc:bc59:ff9f
+22             |10.0.1.166     |2001:cafe::ff07:bdcc:bc59:ff9f
+23             |null           |2001:cafe::d46a:9bdc:8126:b00b
+24             |10.0.1.166     |2001:cafe::13e1:16fc:8726:1bf8
+25             |10.0.1.199     |2001:cafe::ff07:bdcc:bc59:ff9f
+;
+
+//
+// waiting on https://github.com/elastic/elasticsearch/issues/34799
+//
+filterInCIDRIpv4-Ignore
+SELECT id, client_ip, dest_ip FROM logs WHERE dest_ip IN ('10.0.1.1', '10.0.1.200', '10.0.0.0/16') ORDER BY id LIMIT 10;
+
+      id       |   client_ip   |           dest_ip            
+---------------+---------------+------------------------------
+20             |10.0.1.199     |2001:cafe::ff07:bdcc:bc59:ff9f
+22             |10.0.1.166     |2001:cafe::ff07:bdcc:bc59:ff9f
+23             |10.0.1.199     |2001:cafe::d46a:9bdc:8126:b00b
+24             |10.0.1.166     |2001:cafe::13e1:16fc:8726:1bf8
+;
+
+
+filterInCIDRIpv6-Ignore
+SELECT id, client_ip, dest_ip FROM logs WHERE dest_ip IN ('127.0.0.1', '2001:cafe::13e1:16fc:8726:1bf8', '2001:cafe::/48') ORDER BY id LIMIT 10;
+
+      id       |   client_ip   |           dest_ip            
+---------------+---------------+------------------------------
+20             |10.0.1.199     |2001:cafe::ff07:bdcc:bc59:ff9f
+22             |10.0.1.166     |2001:cafe::ff07:bdcc:bc59:ff9f
+23             |10.0.1.199     |2001:cafe::d46a:9bdc:8126:b00b
+24             |10.0.1.166     |2001:cafe::13e1:16fc:8726:1bf8
+;
+
+groupByIpv4
+SELECT client_ip FROM logs GROUP BY client_ip LIMIT 5;
+
+   client_ip:s   
+---------------
+null           
+10.0.0.105     
+10.0.0.107     
+10.0.0.109     
+10.0.0.113    
+;
+
+groupByIpv6
+SELECT dest_ip FROM logs GROUP BY dest_ip ORDER BY dest_ip DESC LIMIT 5;
+
+           dest_ip            
+------------------------------
+fe80::a65e:60ff:fee8:fee9     
+fe80::86ba:3bff:fe05:c3f3     
+2001:cafe::ff07:bdcc:bc59:ff9f
+2001:cafe::ff07:bdcc:bc59:ff9e
+2001:cafe::ff07:bdcc:bc59:ff9d
+;
+
+groupByIpv4AndIpv6
+SELECT client_ip, dest_ip FROM logs GROUP BY client_ip, dest_ip ORDER BY dest_ip DESC LIMIT 5;
+
+   client_ip   |           dest_ip            
+---------------+------------------------------
+10.0.1.222     |fe80::a65e:60ff:fee8:fee9     
+10.0.1.13      |fe80::86ba:3bff:fe05:c3f3     
+null           |2001:cafe::ff07:bdcc:bc59:ff9f
+10.0.1.166     |2001:cafe::ff07:bdcc:bc59:ff9f
+10.0.1.199     |2001:cafe::ff07:bdcc:bc59:ff9f
+;
+
+
+groupByIpv4AndPort
+SELECT client_ip, client_port FROM logs GROUP BY client_ip, client_port ORDER BY client_port DESC LIMIT 5;
+
+   client_ip   |  client_port  
+---------------+---------------
+10.0.1.9       |65004          
+10.0.0.129     |63982          
+null           |63238          
+null           |61337          
+null           |61220       
+;

x-pack/qa/sql/src/main/resources/logs.csv

@@ -0,0 +1,101 @@
+id,@timestamp,bytes_in,bytes_out,client_ip,client_port,dest_ip,status
+1,2017-11-10T21:15:54Z,47,388,10.0.1.1,9152,172.27.1.129,OK
+2,2017-11-10T21:15:39Z,29,374,10.0.1.1,31693,172.27.1.123,OK
+3,2017-11-10T21:15:39Z,35,303,10.0.1.1,23625,172.27.1.113,OK
+4,2017-11-10T21:15:39Z,36,312,10.0.1.1,9932,172.27.1.116,OK
+5,2017-11-10T21:15:40Z,35,344,10.0.1.1,22695,172.27.1.149,OK
+6,2017-11-10T21:15:40Z,31,503,10.0.1.1,59811,172.27.1.122,OK
+7,2017-11-10T21:15:40Z,35,458,10.0.1.7,57372,172.27.1.140,OK
+8,2017-11-10T21:15:41Z,35,281,10.0.1.8,17370,,OK
+9,2017-11-10T21:15:41Z,46,231,10.0.1.9,65004,,OK
+10,2017-11-10T20:36:07Z,40,506,10.0.1.10,22661,,OK
+11,2017-11-10T20:36:08Z,34,471,10.0.1.11,16752,172.27.1.131,OK
+12,2017-11-10T20:36:07Z,39,503,10.0.1.12,19479,172.27.1.103,OK
+13,2017-11-10T20:36:07Z,29,502,10.0.1.13,2326,172.27.1.139,OK
+14,2017-11-10T20:36:15Z,35,280,10.0.1.13,51758,172.27.1.129,OK
+15,2017-11-10T20:36:15Z,38,225,,22994,172.27.1.139,OK
+16,2017-11-10T20:35:54Z,35,326,,5505,172.27.1.120,OK
+17,2017-11-10T20:35:54Z,46,466,10.0.1.13,3666,172.27.1.103,OK
+18,2017-11-10T20:35:55Z,42,238,10.0.1.13,23791,172.27.1.111,OK
+19,2017-11-10T17:54:43Z,16,,10.0.1.13,,fe80::86ba:3bff:fe05:c3f3,OK
+20,2017-11-10T23:23:24Z,40,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+21,2017-11-10T17:54:59Z,24,,10.0.1.222,,fe80::a65e:60ff:fee8:fee9,OK
+22,2017-11-10T21:13:27Z,20,,10.0.1.166,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+23,2017-11-10T22:37:41Z,24,,,,2001:cafe::d46a:9bdc:8126:b00b,OK
+24,2017-11-10T20:34:43Z,8,,10.0.1.166,,2001:cafe::13e1:16fc:8726:1bf8,OK
+25,2017-11-10T23:30:46Z,40,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+26,2017-11-10T21:13:16Z,20,,,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+27,2017-11-10T23:36:32Z,0,,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK
+28,2017-11-10T23:36:33Z,40,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+29,2017-11-10T20:35:26Z,20,,10.0.1.166,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+30,2017-11-10T23:36:41Z,8,,,,2001:cafe::13e1:16fc:8726:1bf8,OK
+31,2017-11-10T23:56:36Z,8,,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK
+32,2017-11-10T20:29:25Z,32,,10.0.1.177,59769,172.20.10.1,Error
+33,2017-11-10T21:35:01Z,20,,10.0.1.166,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+34,2017-11-10T21:12:17Z,20,,10.0.1.166,,2001:cafe::ff07:bdcc:bc59:ff9e,OK
+35,2017-11-10T23:17:14Z,40,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9d,OK
+36,2017-11-10T23:28:11Z,8,,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK
+37,2017-11-10T22:36:27Z,8,,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK
+38,2017-11-10T20:35:55Z,36,281,,58533,172.27.1.1,OK
+39,2017-11-10T20:35:55Z,25,273,,39211,,OK
+40,2017-11-10T20:35:55Z,34,253,,37971,172.27.1.1,OK
+41,2017-11-10T20:35:55Z,41,503,,47831,172.27.1.1,OK
+42,2017-11-10T21:34:49Z,28,,,,27.58.6.220,Error
+43,2017-11-10T20:35:55Z,28,206,10.0.1.200,31000,172.27.1.1,OK
+44,2017-11-10T20:14:04Z,8,,10.0.1.201,,2001:cafe::13e1:16fc:8726:1bf8,OK
+45,2017-11-10T19:38:06Z,37,239,10.0.1.202,3577,172.27.1.1,OK
+46,2017-11-10T21:14:18Z,8,,10.0.1.203,,2001:cafe::13e1:16fc:8726:1bf8,OK
+47,2017-11-10T20:35:56Z,34,202,10.0.1.204,49112,172.27.1.1,OK
+48,2017-11-10T20:53:05Z,8,,10.0.1.205,,2001:cafe::13e1:16fc:8726:1bf8,OK
+49,2017-11-10T21:25:42Z,8,,10.0.1.206,,2001:cafe::13e1:16fc:8726:1bf9,OK
+50,2017-11-10T21:14:44Z,8,,10.0.1.207,,2001:cafe::13e1:16fc:8726:1bf4,OK
+51,2017-11-10T21:28:34Z,8,,10.0.1.208,,2001:cafe::13e1:16fc:8726:1bf3,OK
+52,2017-11-10T20:35:55Z,34,227,,63238,172.27.1.1,OK
+53,2017-11-10T20:15:24Z,8,,,,2001:cafe::13e1:16fc:8726:1bf8,OK
+54,2017-11-10T20:35:57Z,37,239,,61337,172.27.1.1,OK
+55,2017-11-10T17:14:10Z,16,,10.0.1.222,,2001:cafe::a98d:374:79e4:4865,OK
+56,2017-11-10T20:35:57Z,38,476,10.0.1.200,53720,172.27.1.1,OK
+57,2017-11-10T23:22:13Z,8,,10.0.1.201,,2001:cafe::13e1:16fc:8726:1bf8,OK
+58,2017-11-10T20:32:57Z,8,,10.0.1.202,,2001:cafe::13e1:16fc:8726:1bf7,OK
+59,2017-11-10T21:24:00Z,8,,10.0.1.203,,2001:cafe::13e1:16fc:8726:1bf6,OK
+60,2017-11-10T20:35:56Z,32,503,10.0.1.204,19382,172.27.1.1,OK
+61,2017-11-10T23:43:10Z,0,,10.0.1.205,,2001:cafe::13e1:16fc:8726:1bf8,OK
+62,2017-11-10T20:35:57Z,30,169,10.0.1.206,47532,172.27.1.1,OK
+63,2017-11-10T20:21:58Z,20,,10.0.1.207,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+64,2017-11-10T20:35:57Z,41,271,10.0.1.208,16227,,OK
+65,2017-11-10T20:33:06Z,28,,10.0.1.166,,172.27.1.1,Error
+66,2017-11-10T20:35:57Z,33,185,,28928,172.27.1.1,OK
+67,2017-11-10T20:26:21Z,20,,,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+68,2017-11-10T21:23:25Z,20,,,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+69,2017-11-10T21:23:54Z,8,,10.0.1.166,,2001:cafe::13e1:16fc:8726:1bf8,OK
+70,2017-11-10T20:35:57Z,35,234,10.0.1.166,54994,172.27.1.1,OK
+71,2017-11-10T00:27:03Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b6,OK
+72,2017-11-10T00:27:46Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b6,OK
+73,2017-11-10T20:35:58Z,35,223,,20163,172.27.1.1,OK
+74,2017-11-10T20:35:57Z,32,501,10.0.1.166,51275,172.27.1.1,OK
+75,2017-11-10T22:27:09Z,20,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+76,2017-11-10T20:35:58Z,45,493,10.0.1.166,1193,172.27.1.1,OK
+77,2017-11-10T22:26:44Z,20,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK
+78,2017-11-10T22:27:31Z,8,,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK
+79,2017-11-10T20:35:52Z,47,246,,24564,172.27.1.1,OK
+80,2017-11-10T00:00:22Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b6,OK
+81,2017-11-10T20:35:52Z,37,420,10.0.1.166,40542,172.27.1.1,OK
+82,2017-11-10T00:01:20Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b6,OK
+83,2017-11-10T00:01:04Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b6,OK
+84,2017-11-10T00:32:48Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b5,OK
+85,2017-11-10T00:01:45Z,48,,10.0.1.122,,2001:cafe::470f:60b7:f84a:25b4,OK
+86,2017-11-10T20:36:08Z,38,509,,61220,172.27.1.1,OK
+87,2017-11-10T21:17:37Z,38,226,10.0.0.144,26602,,OK
+88,2017-11-10T20:06:49Z,30,,10.0.0.147,53240,90.128.199.24,Error
+89,2017-11-10T21:17:37Z,44,284,10.0.0.118,49479,172.27.1.1,OK
+90,2017-11-10T19:51:38Z,28,,10.0.0.130,,203.131.98.151,Error
+91,2017-11-10T19:51:38Z,28,,10.0.0.107,,172.20.10.8,Error
+92,2017-11-10T20:06:50Z,34,215,10.0.0.113,25162,90.128.199.24,OK
+93,2017-11-10T21:17:46Z,33,185,10.0.0.129,63982,172.27.1.1,OK
+94,2017-11-10T19:51:38Z,28,,10.0.0.130,,203.131.98.151,Error
+95,2017-11-10T21:17:46Z,28,321,10.0.0.105,4292,172.27.1.1,OK
+96,2017-11-10T00:04:50Z,48,,10.0.0.109,,2001:cafe::470f:60b7:f84a:25b6,OK
+97,2017-11-10T21:17:48Z,30,280,10.0.0.145,57783,172.27.1.1,OK
+98,2017-11-10T21:12:24Z,74,90,10.0.0.134,57203,172.20.10.1,OK
+99,2017-11-10T21:17:37Z,39,512,10.0.0.128,29333,,OK
+100,2017-11-10T03:21:36Z,64,183,10.0.0.129,4541,172.16.1.1,OK