EQL: Add integration tests harness to test EQL feature parity with original implementation (#52248) (#52675)

The tests use the original test queries from
https://github.com/endgameinc/eql/blob/master/eql/etc/test_queries.toml
for EQL implementation correctness validation.
The file test_queries_unsupported.toml serves as a "blacklist" for the
queries that we do not support. Currently all of the queries are
blacklisted. Over the time the expectation is to eventually have an
empty "blacklist" when all of the queries are fully supported.

The tests use the original test vector from
https://raw.githubusercontent.com/endgameinc/eql/master/eql/etc/test_data.json.

Only one EQL and the response is stubbed for now to match the expected
output from that query. This part would need some tweaking after EQL is
fully wired.

Related to https://github.com/elastic/elasticsearch/issues/49581

client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java

@@ -40,9 +40,9 @@ public class EqlIT extends ESRestHighLevelClientTestCase {
         assertNotNull(response);
         assertFalse(response.isTimeout());
         assertNotNull(response.hits());
-        assertNull(response.hits().events());
+        assertNull(response.hits().sequences());
         assertNull(response.hits().counts());
-        assertNotNull(response.hits().sequences());
-        assertThat(response.hits().sequences().size(), equalTo(2));
+        assertNotNull(response.hits().events());
+        assertThat(response.hits().events().size(), equalTo(1));
     }
 }

x-pack/plugin/eql/build.gradle

@@ -17,6 +17,18 @@ ext {
 
 archivesBaseName = 'x-pack-eql'
 
+// All integration tests live in qa modules
+integTest.enabled = false
+
+// Instead we create a separate task to run the tests based on ESIntegTestCase
+task internalClusterTest(type: Test) {
+  mustRunAfter test
+  include '**/*IT.class'
+  systemProperty 'es.set.netty.runtime.available.processors', 'false'
+}
+
+check.dependsOn internalClusterTest
+
 dependencies {
   compileOnly project(path: xpackModule('core'), configuration: 'default')
   compileOnly(project(':modules:lang-painless')) {
@@ -31,21 +43,17 @@ dependencies {
   testCompile project(path: ':modules:reindex', configuration: 'runtime')
   testCompile project(path: ':modules:parent-join', configuration: 'runtime')
   testCompile project(path: ':modules:analysis-common', configuration: 'runtime')
+
+  // TOML parser for EqlActionIT tests
+  testCompile 'io.ous:jtoml:2.0.0'
+
+  // JSON parser for tests input data
+  testCompile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+  testCompile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+  testCompile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
+
 }
 
-integTest.enabled = false
-testingConventions.enabled = false
-
-// Instead we create a separate task to run the tests based on ESIntegTestCase
-task internalClusterTest(type: Test) {
-  description = '🌈🌈🌈🦄 Welcome to fantasy integration tests land! 🦄🌈🌈🌈'
-  mustRunAfter test
-
-  include '**/*IT.class'
-  systemProperty 'es.set.netty.runtime.available.processors', 'false'
-}
-
-check.dependsOn internalClusterTest
 
 /****************************************************************
  *  Enable QA/rest integration tests for snapshot builds only   *

x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/10_basic.yml

@@ -23,5 +23,5 @@ setup:
 
   - match: {timed_out: false}
   - match: {took: 0}
-  - match: {hits.total.value: 0}
+  - match: {hits.total.value: 1}
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.action;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.index.query.QueryBuilder;
+
+public class EqlSearchRequestBuilder extends ActionRequestBuilder<EqlSearchRequest, EqlSearchResponse> {
+    public EqlSearchRequestBuilder(ElasticsearchClient client, EqlSearchAction action) {
+        super(client, action, new EqlSearchRequest());
+    }
+
+    public EqlSearchRequestBuilder indices(String... indices) {
+        request.indices(indices);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder query(QueryBuilder query) {
+        request.query(query);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder timestampField(String timestampField) {
+        request.timestampField(timestampField);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder eventTypeField(String eventTypeField) {
+        request.eventTypeField(eventTypeField);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder implicitJoinKeyField(String implicitJoinKeyField) {
+        request.implicitJoinKeyField(implicitJoinKeyField);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder fetchSize(int size) {
+        request.fetchSize(size);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder searchAfter(Object[] values) {
+        request.searchAfter(values);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder rule(String rule) {
+        request.rule(rule);
+        return this;
+    }
+
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java

@@ -27,50 +27,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
-
-/**
- * Response to perform an eql search
- *
- * Example events response:
- *         List<SearchHit> events = Arrays.asList(
- *             new SearchHit(1, "111", null),
- *             new SearchHit(2, "222", null)
- *         );
- *         EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(Arrays.asList(
- *             new EqlSearchResponse.Sequence(Collections.singletonList("4021"), events),
- *             new EqlSearchResponse.Sequence(Collections.singletonList("2343"), events)
- *         ), null, null, new TotalHits(0, TotalHits.Relation.EQUAL_TO));
- *         EqlSearchResponse response = new EqlSearchResponse(hits, 5, false);
- *
- *
- *  Example sequence response:
- *         List<SearchHit> events1 = Arrays.asList(
- *             new SearchHit(1, "111", null),
- *             new SearchHit(2, "222", null)
- *         );
- *         List<SearchHit> events2 = Arrays.asList(
- *             new SearchHit(1, "333", null),
- *             new SearchHit(2, "444", null)
- *         );
- *         List<Sequence> sequences = Arrays.asList(
- *                 new EqlSearchResponse.Sequence(new String[]{"4021"}, events1),
- *                 new EqlSearchResponse.Sequence(new String[]{"2343"}, events2)
- *             );
- *
- *         EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(null, sequences, null, new TotalHits(100, TotalHits.Relation.EQUAL_TO));
- *         EqlSearchResponse response = new EqlSearchResponse(hits, 5, false);
- *
- *
- *  Example count response:
- *         TotalHits totals = new TotalHits(100, TotalHits.Relation.EQUAL_TO);
- *         List<Count> counts = Arrays.asList(
- *                 new EqlSearchResponse.Count(40, new String[]{"foo", "bar"}, .42233f),
- *                 new EqlSearchResponse.Count(15, new String[]{"foo", "bar"}, .170275f)
- *         );
- *
- *         EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(null, null, counts, totals);
- *         EqlSearchResponse response = new EqlSearchResponse(hits, 5, false);
- */
 public class EqlSearchResponse extends ActionResponse implements ToXContentObject {
 
     private final Hits hits;
@@ -399,7 +355,7 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec
             } else {
                 totalHits = null;
             }
-            events =  in.readBoolean() ? in.readList(SearchHit::new) : null;
+            events = in.readBoolean() ? in.readList(SearchHit::new) : null;
             sequences = in.readBoolean() ? in.readList(Sequence::new) : null;
             counts = in.readBoolean() ? in.readList(Count::new) : null;
         }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java

@@ -22,6 +22,7 @@ import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
@@ -45,6 +46,8 @@ import java.util.function.Supplier;
 
 public class EqlPlugin extends Plugin implements ActionPlugin {
 
+    private final boolean enabled;
+
     private static final boolean EQL_FEATURE_FLAG_REGISTERED;
 
     static {
@@ -69,16 +72,20 @@ public class EqlPlugin extends Plugin implements ActionPlugin {
         Setting.Property.NodeScope
     );
 
+    public EqlPlugin(final Settings settings) {
+        this.enabled = EQL_ENABLED_SETTING.get(settings);
+    }
+
     @Override
     public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
             ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
             Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
             IndexNameExpressionResolver expressionResolver) {
-
         return createComponents(client, clusterService.getClusterName().value(), namedWriteableRegistry);
     }
 
-    private Collection<Object> createComponents(Client client, String clusterName, NamedWriteableRegistry namedWriteableRegistry) {
+    private Collection<Object> createComponents(Client client, String clusterName,
+                                                NamedWriteableRegistry namedWriteableRegistry) {
         IndexResolver indexResolver = new IndexResolver(client, clusterName, DefaultDataTypeRegistry.INSTANCE);
         PlanExecutor planExecutor = new PlanExecutor(client, indexResolver, namedWriteableRegistry);
         return Arrays.asList(planExecutor);
@@ -91,14 +98,6 @@ public class EqlPlugin extends Plugin implements ActionPlugin {
         return modules;
     }
 
-    @Override
-    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
-        return Arrays.asList(
-            new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class),
-            new ActionHandler<>(EqlStatsAction.INSTANCE, TransportEqlStatsAction.class)
-        );
-    }
-
     /**
      * The settings defined by EQL plugin.
      *
@@ -113,6 +112,17 @@ public class EqlPlugin extends Plugin implements ActionPlugin {
         }
     }
 
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        if (enabled) {
+            return Arrays.asList(
+                new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class),
+                new ActionHandler<>(EqlStatsAction.INSTANCE, TransportEqlStatsAction.class)
+            );
+        }
+        return Collections.emptyList();
+    }
+
     boolean isSnapshot() {
         return Build.CURRENT.isSnapshot();
     }
@@ -131,9 +141,14 @@ public class EqlPlugin extends Plugin implements ActionPlugin {
                                              IndexNameExpressionResolver indexNameExpressionResolver,
                                              Supplier<DiscoveryNodes> nodesInCluster) {
 
-        if (isEnabled(settings) == false) {
-            return Collections.emptyList();
+        if (enabled) {
+            return Arrays.asList(new RestEqlSearchAction(), new RestEqlStatsAction());
         }
-        return Arrays.asList(new RestEqlSearchAction(), new RestEqlStatsAction());
+        return Collections.emptyList();
+    }
+
+    // overridable by tests
+    protected XPackLicenseState getLicenseState() {
+        return XPackPlugin.getSharedLicenseState();
     }
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java

@@ -31,7 +31,6 @@ import org.elasticsearch.xpack.eql.session.Results;
 
 import java.time.ZoneId;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 
 public class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRequest, EqlSearchResponse> {
@@ -41,7 +40,7 @@ public class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRe
 
     @Inject
     public TransportEqlSearchAction(Settings settings, ClusterService clusterService, TransportService transportService,
-            ThreadPool threadPool, ActionFilters actionFilters, PlanExecutor planExecutor) {
+                                    ThreadPool threadPool, ActionFilters actionFilters, PlanExecutor planExecutor) {
         super(EqlSearchAction.NAME, transportService, actionFilters, EqlSearchRequest::new);
 
         this.securityContext = XPackSettings.SECURITY_ENABLED.get(settings) ?
@@ -56,19 +55,19 @@ public class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRe
     }
 
     public static void operation(PlanExecutor planExecutor, EqlSearchRequest request, String username,
-            String clusterName, ActionListener<EqlSearchResponse> listener) {
+                                 String clusterName, ActionListener<EqlSearchResponse> listener) {
         // TODO: these should be sent by the client
         ZoneId zoneId = DateUtils.of("Z");
         QueryBuilder filter = request.query();
         TimeValue timeout = TimeValue.timeValueSeconds(30);
         boolean includeFrozen = request.indicesOptions().ignoreThrottled() == false;
         String clientId = null;
-        
+
         ParserParams params = new ParserParams()
-                .fieldEventType(request.eventTypeField())
-                .fieldTimestamp(request.timestampField())
-                .implicitJoinKey(request.implicitJoinKeyField());
-        
+            .fieldEventType(request.eventTypeField())
+            .fieldTimestamp(request.timestampField())
+            .implicitJoinKey(request.implicitJoinKeyField());
+
         Configuration cfg = new Configuration(request.indices(), zoneId, username, clusterName, filter, timeout, includeFrozen, clientId);
         //planExecutor.eql(cfg, request.rule(), params, wrap(r -> listener.onResponse(createResponse(r)), listener::onFailure));
         listener.onResponse(createResponse(null));
@@ -77,14 +76,14 @@ public class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRe
     static EqlSearchResponse createResponse(Results results) {
         // Stubbed search response
         // TODO: implement actual search response processing once the parser/executor is in place
+        // Updated for stubbed response to: process where serial_event_id = 1
+        // to validate the sample test until the engine is wired in.
         List<SearchHit> events = Arrays.asList(
-            new SearchHit(1, "111", null, null),
-            new SearchHit(2, "222", null, null)
+            new SearchHit(1, "111", null, null)
         );
-        EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(null, Arrays.asList(
-            new EqlSearchResponse.Sequence(Collections.singletonList("4021"), events),
-            new EqlSearchResponse.Sequence(Collections.singletonList("2343"), events)
-        ), null, new TotalHits(0, TotalHits.Relation.EQUAL_TO));
+        EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(events, null,
+            null, new TotalHits(1, TotalHits.Relation.EQUAL_TO));
+
         return new EqlSearchResponse(hits, 0, false);
     }
 
@@ -95,4 +94,4 @@ public class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRe
     static String clusterName(ClusterService clusterService) {
         return clusterService.getClusterName().value();
     }
-}
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java

@@ -16,19 +16,20 @@ import static org.elasticsearch.test.ESTestCase.randomZone;
 
 public final class EqlTestUtils {
 
-    private EqlTestUtils() {}
+    private EqlTestUtils() {
+    }
 
-    public static final Configuration TEST_CFG = new Configuration(new String[] { "none" }, org.elasticsearch.xpack.ql.util.DateUtils.UTC,
-            "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, "");
+    public static final Configuration TEST_CFG = new Configuration(new String[]{"none"}, org.elasticsearch.xpack.ql.util.DateUtils.UTC,
+        "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, "");
 
     public static Configuration randomConfiguration() {
-        return new Configuration(new String[] {randomAlphaOfLength(16)},
-                randomZone(),
-                randomAlphaOfLength(16),
-                randomAlphaOfLength(16),
-                null,
-                new TimeValue(randomNonNegativeLong()),
-                randomBoolean(),
-                randomAlphaOfLength(16));
+        return new Configuration(new String[]{randomAlphaOfLength(16)},
+            randomZone(),
+            randomAlphaOfLength(16),
+            randomAlphaOfLength(16),
+            null,
+            new TimeValue(randomNonNegativeLong()),
+            randomBoolean(),
+            randomAlphaOfLength(16));
     }
 }

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.action;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.LicenseService;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.eql.plugin.EqlPlugin;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
+
+@ESIntegTestCase.ClusterScope(scope = SUITE, numDataNodes = 0, numClientNodes = 0, maxNumDataNodes = 0)
+public abstract class AbstractEqlIntegTestCase extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal));
+        settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false);
+        settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
+        settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
+        settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
+        settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
+        settings.put(EqlPlugin.EQL_ENABLED_SETTING.getKey(), true);
+        settings.put(LicenseService.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
+        return settings.build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singletonList(LocalStateEqlXPackPlugin.class);
+    }
+}
+

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.action;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.elasticsearch.Build;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.SearchHit;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class EqlActionIT extends AbstractEqlIntegTestCase {
+
+    static final String indexPrefix = "endgame";
+    static final String testIndexName = indexPrefix + "-1.4.0";
+    protected static final String PARAM_FORMATTING = "%1$s.test";
+
+
+    @BeforeClass
+    public static void checkForSnapshot() {
+        assumeTrue("Only works on snapshot builds for now", Build.CURRENT.isSnapshot());
+    }
+
+    @Before
+    public void setUpData() throws Exception {
+        // Insert test data
+        ObjectMapper mapper = new ObjectMapper();
+        BulkRequestBuilder bulkBuilder = client().prepareBulk();
+        JsonNode rootNode = mapper.readTree(EqlActionIT.class.getResourceAsStream("/test_data.json"));
+        Iterator<JsonNode> entries = rootNode.elements();
+        while (entries.hasNext()) {
+            JsonNode entry = entries.next();
+            bulkBuilder.add(new IndexRequest(testIndexName).source(entry.toString(), XContentType.JSON));
+        }
+        BulkResponse bulkResponse = bulkBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
+        assertThat(bulkResponse.hasFailures() ? bulkResponse.buildFailureMessage() : "", bulkResponse.hasFailures(), equalTo(false));
+
+        ensureYellow(testIndexName);
+    }
+
+    @After
+    public void tearDownData() {
+        client().admin().indices().prepareDelete(testIndexName).get();
+    }
+
+    @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING)
+    public static List<Object[]> readTestSpecs() throws Exception {
+        List<Object[]> testSpecs = new ArrayList<>();
+
+        // Load EQL validation specs
+        List<EqlSpec> specs = EqlSpecLoader.load("/test_queries.toml", true);
+        List<EqlSpec> unsupportedSpecs = EqlSpecLoader.load("/test_queries_unsupported.toml", false);
+
+        // Validate only currently supported specs
+        int num = 1; // Seq number of the test
+        for (EqlSpec spec : specs) {
+            boolean supported = true;
+            // Check if spec is supported, simple iteration, cause the list is short.
+            for (EqlSpec unSpec : unsupportedSpecs) {
+                if (spec.query() != null && spec.query().equals(unSpec.query())) {
+                    supported = false;
+                    break;
+                }
+            }
+
+            if (supported) {
+                testSpecs.add(new Object[]{num++, spec});
+            }
+        }
+        return testSpecs;
+    }
+
+    private final int num;
+    private final EqlSpec spec;
+
+    public EqlActionIT(int num, EqlSpec spec) {
+        this.num = num;
+        this.spec = spec;
+    }
+
+    public final void test() {
+        EqlSearchResponse response = new EqlSearchRequestBuilder(client(), EqlSearchAction.INSTANCE)
+            .indices(testIndexName).rule(spec.query()).get();
+
+        List<SearchHit> events = response.hits().events();
+        assertNotNull(events);
+
+        final int len = events.size();
+        final long ids[] = new long[len];
+        for (int i = 0; i < events.size(); i++) {
+            ids[i] = events.get(i).docId();
+        }
+        final String msg = "unexpected result for spec: [" + spec.toString() + "]";
+        assertArrayEquals(msg, spec.expectedEventIds(), ids);
+    }
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSpec.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.action;
+
+import org.elasticsearch.common.Strings;
+
+import java.util.Arrays;
+
+public class EqlSpec {
+    private String description;
+    private String note;
+    private String[] tags;
+    private String query;
+    private long[] expectedEventIds;
+
+    public String description() {
+        return description;
+    }
+
+    public void description(String description) {
+        this.description = description;
+    }
+
+    public String note() {
+        return note;
+    }
+
+    public void note(String note) {
+        this.note = note;
+    }
+
+    public String[] tags() {
+        return tags;
+    }
+
+    public void tags(String[] tags) {
+        this.tags = tags;
+    }
+
+    public String query() {
+        return query;
+    }
+
+    public void query(String query) {
+        this.query = query;
+    }
+
+    public long[] expectedEventIds() {
+        return expectedEventIds;
+    }
+
+    public void expectedEventIds(long[] expectedEventIds) {
+        this.expectedEventIds = expectedEventIds;
+    }
+
+    @Override
+    public String toString() {
+        String str = "";
+        str = appendWithComma(str, "query", query);
+        str = appendWithComma(str, "description", description);
+        str = appendWithComma(str, "note", note);
+
+        if (tags != null) {
+            str = appendWithComma(str, "tags", Arrays.toString(tags));
+        }
+
+        if (expectedEventIds != null) {
+            str = appendWithComma(str, "expected_event_ids", Arrays.toString(expectedEventIds));
+        }
+        return str;
+    }
+
+    private static String appendWithComma(String str, String name, String append) {
+        if (!Strings.isNullOrEmpty(append)) {
+            if (!Strings.isNullOrEmpty(str)) {
+                str += ", ";
+            }
+            str += name + ": " + append;
+        }
+        return str;
+    }
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSpecLoader.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.action;
+
+import io.ous.jtoml.JToml;
+import io.ous.jtoml.Toml;
+import io.ous.jtoml.TomlTable;
+import org.elasticsearch.common.Strings;
+
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+public class EqlSpecLoader {
+    public static List<EqlSpec> load(String path, boolean supported) throws Exception {
+        try (InputStream is = EqlSpecLoader.class.getResourceAsStream(path)) {
+            return readFromStream(is, supported);
+        }
+    }
+
+    private static void validateAndAddSpec(List<EqlSpec> specs, EqlSpec spec, boolean supported) throws Exception {
+        if (Strings.isNullOrEmpty(spec.query())) {
+            throw new IllegalArgumentException("Read a test without a query value");
+        }
+
+        if (supported && spec.expectedEventIds() == null) {
+            throw new IllegalArgumentException("Read a test without a expected_event_ids value");
+        }
+
+        specs.add(spec);
+    }
+
+    private static String getTrimmedString(TomlTable table, String key) {
+        String s = table.getString(key);
+        if (s != null) {
+            return s.trim();
+        }
+        return null;
+    }
+
+    private static List<EqlSpec> readFromStream(InputStream is, boolean supported) throws Exception {
+        List<EqlSpec> testSpecs = new ArrayList<>();
+
+        EqlSpec spec;
+        Toml toml = JToml.parse(is);
+
+        List<TomlTable> queries = toml.getArrayTable("queries");
+        for (TomlTable table : queries) {
+            spec = new EqlSpec();
+            spec.query(getTrimmedString(table, "query"));
+            spec.note(getTrimmedString(table, "note"));
+            spec.description(getTrimmedString(table, "description"));
+
+            List<?> arr = table.getList("tags");
+            if (arr != null) {
+                String tags[] = new String[arr.size()];
+                int i = 0;
+                for (Object obj : arr) {
+                    tags[i] = (String) obj;
+                }
+                spec.tags(tags);
+            }
+
+            arr = table.getList("expected_event_ids");
+            if (arr != null) {
+                long expectedEventIds[] = new long[arr.size()];
+                int i = 0;
+                for (Object obj : arr) {
+                    expectedEventIds[i++] = (Long) obj;
+                }
+                spec.expectedEventIds(expectedEventIds);
+            }
+            validateAndAddSpec(testSpecs, spec, supported);
+        }
+
+        return testSpecs;
+    }
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/LocalStateEqlXPackPlugin.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.action;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.eql.plugin.EqlPlugin;
+
+import java.nio.file.Path;
+
+public class LocalStateEqlXPackPlugin extends LocalStateCompositeXPackPlugin {
+
+    public LocalStateEqlXPackPlugin(final Settings settings, final Path configPath) throws Exception {
+        super(settings, configPath);
+        LocalStateEqlXPackPlugin thisVar = this;
+        plugins.add(new EqlPlugin(settings) {
+            @Override
+            protected XPackLicenseState getLicenseState() {
+                return thisVar.getLicenseState();
+            }
+        });
+    }
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/plugin/EqlPluginTests.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.eql.plugin;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.hasItem;
@@ -13,7 +14,7 @@ import static org.hamcrest.Matchers.not;
 
 public class EqlPluginTests extends ESTestCase {
     public void testEnabledSettingRegisteredInSnapshotBuilds() {
-        final EqlPlugin plugin = new EqlPlugin() {
+        final EqlPlugin plugin = new EqlPlugin(Settings.EMPTY) {
 
             @Override
             protected boolean isSnapshot() {
@@ -25,7 +26,7 @@ public class EqlPluginTests extends ESTestCase {
     }
 
     public void testEnabledSettingNotRegisteredInNonSnapshotBuilds() {
-        final EqlPlugin plugin = new EqlPlugin() {
+        final EqlPlugin plugin = new EqlPlugin(Settings.EMPTY) {
 
             @Override
             protected boolean isSnapshot() {