Remove x-pack from build, distribution and packaging. (#43)

This PR removes references to x-pack from buildSrc, distribution and qa modules.

Signed-off-by: Peter Nied <petern@amazon.com>

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -58,8 +58,7 @@ class PluginBuildPlugin implements Plugin<Project> {
         PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project)
         configureDependencies(project)
 
-        boolean isXPackModule = project.path.startsWith(':x-pack:plugin')
+        boolean isModule = project.path.startsWith(':modules:')
-        boolean isModule = project.path.startsWith(':modules:') || isXPackModule
 
         createBundleTasks(project, extension)
 
@@ -103,7 +102,7 @@ class PluginBuildPlugin implements Plugin<Project> {
                 expand(properties)
                 inputs.properties(properties)
             }
-            if (isModule == false || isXPackModule) {
+            if (isModule == false) {
                 addNoticeGeneration(project, extension1)
             }
         }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -489,11 +489,6 @@ public class ElasticsearchNode implements TestClusterConfiguration {
             pluginsToInstall.addAll(plugins.stream().map(Provider::get).map(p -> p.toURI().toString()).collect(Collectors.toList()));
         }
 
-        if (requiresAddingXPack()) {
-            logToProcessStdout("emulating the " + testDistribution + " flavor for " + getVersion() + " by installing x-pack");
-            pluginsToInstall.add("x-pack");
-        }
-
         if (pluginsToInstall.isEmpty() == false) {
             if (getVersion().onOrAfter("7.6.0")) {
                 logToProcessStdout("installing " + pluginsToInstall.size() + " plugins in a single transaction");
@@ -554,18 +549,10 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         startElasticsearchProcess();
     }
 
-    private boolean requiresAddingXPack() {
-        return getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT;
-    }
-
     private boolean canUseSharedDistribution() {
         // using original location can be too long due to MAX_PATH restrictions on windows CI
         // TODO revisit when moving to shorter paths on CI by using Teamcity
-        return OS.current() != OS.WINDOWS
+        return OS.current() != OS.WINDOWS && extraJarFiles.size() == 0 && modules.size() == 0 && plugins.size() == 0;
-            && extraJarFiles.size() == 0
-            && modules.size() == 0
-            && plugins.size() == 0
-            && requiresAddingXPack() == false;
     }
 
     private void logToProcessStdout(String message) {

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -241,10 +241,6 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
         for (final String pluginId : pluginIds) {
             terminal.println("-> Installing " + pluginId);
             try {
-                if ("x-pack".equals(pluginId)) {
-                    handleInstallXPack(buildFlavor());
-                }
-
                 final List<Path> deleteOnFailure = new ArrayList<>();
                 deleteOnFailures.put(pluginId, deleteOnFailure);
 
@@ -281,24 +277,6 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
         }
     }
 
-    Build.Flavor buildFlavor() {
-        return Build.CURRENT.flavor();
-    }
-
-    private static void handleInstallXPack(final Build.Flavor flavor) throws UserException {
-        switch (flavor) {
-            case DEFAULT:
-                throw new UserException(ExitCodes.CONFIG, "this distribution of Elasticsearch contains X-Pack by default");
-            case OSS:
-                throw new UserException(
-                    ExitCodes.CONFIG,
-                    "X-Pack is not available with the oss distribution; to use X-Pack features use the default distribution"
-                );
-            case UNKNOWN:
-                throw new IllegalStateException("your distribution is broken");
-        }
-    }
-
     /** Downloads the plugin and returns the file it was downloaded to. */
     private Path download(Terminal terminal, String pluginId, Path tmpDir, boolean isBatch) throws Exception {
         if (OFFICIAL_PLUGINS.contains(pluginId)) {

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -525,15 +525,6 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertInstallCleaned(env.v2());
     }
 
-    public void testBuiltinXpackModule() throws Exception {
-        Tuple<Path, Environment> env = createEnv(fs, temp);
-        Path pluginDir = createPluginDir(temp);
-        String pluginZip = createPluginUrl("x-pack", pluginDir);
-        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("is a system module"));
-        assertInstallCleaned(env.v2());
-    }
-
     public void testJarHell() throws Exception {
         // jar hell test needs a real filesystem
         assumeTrue("real filesystem", isReal);
@@ -813,33 +804,6 @@ public class InstallPluginCommandTests extends ESTestCase {
         }
     }
 
-    public void testInstallXPack() throws IOException {
-        runInstallXPackTest(Build.Flavor.DEFAULT, UserException.class, "this distribution of Elasticsearch contains X-Pack by default");
-        runInstallXPackTest(
-            Build.Flavor.OSS,
-            UserException.class,
-            "X-Pack is not available with the oss distribution; to use X-Pack features use the default distribution"
-        );
-        runInstallXPackTest(Build.Flavor.UNKNOWN, IllegalStateException.class, "your distribution is broken");
-    }
-
-    private <T extends Exception> void runInstallXPackTest(final Build.Flavor flavor, final Class<T> clazz, final String expectedMessage)
-        throws IOException {
-        final InstallPluginCommand flavorCommand = new InstallPluginCommand() {
-            @Override
-            Build.Flavor buildFlavor() {
-                return flavor;
-            }
-        };
-
-        final Environment environment = createEnv(fs, temp).v2();
-        final T exception = expectThrows(
-            clazz,
-            () -> flavorCommand.execute(terminal, Collections.singletonList("x-pack"), false, environment)
-        );
-        assertThat(exception, hasToString(containsString(expectedMessage)));
-    }
-
     public void testInstallMisspelledOfficialPlugins() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
 

gradle/formatting.gradle

@@ -119,30 +119,7 @@ def projectPathsToExclude = [
   ':test:fixtures:old-elasticsearch',
   ':test:fixtures:s3-fixture',
   ':test:framework',
-  ':test:logger-usage',
+  ':test:logger-usage'
-  ':x-pack:license-tools',
-  ':x-pack:plugin:analytics',
-  ':x-pack:plugin:core',
-  ':x-pack:plugin:deprecation',
-  ':x-pack:plugin:frozen-indices',
-  ':x-pack:plugin:identity-provider',
-  ':x-pack:plugin:mapper-constant-keyword',
-  ':x-pack:plugin:mapper-flattened',
-  ':x-pack:plugin:ql',
-  ':x-pack:plugin:search-business-rules',
-  ':x-pack:plugin:spatial',
-  ':x-pack:plugin:vectors',
-  ':x-pack:plugin:voting-only-node',
-  ':x-pack:plugin:wildcard',
-  ':x-pack:qa',
-  ':x-pack:qa:security-example-spi-extension',
-  ':x-pack:snapshot-tool',
-  ':x-pack:snapshot-tool:qa:google-cloud-storage',
-  ':x-pack:snapshot-tool:qa:s3',
-  ':x-pack:test:feature-aware',
-  ':x-pack:test:idp-fixture',
-  ':x-pack:test:smb-fixture',
-  ':x-pack:transport-client'
 ]
 
 subprojects {

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -67,7 +67,6 @@ import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
@@ -109,20 +108,6 @@ public class DockerTests extends PackagingTestCase {
         verifyContainerInstallation(installation, distribution());
     }
 
-    /**
-     * Check that the /_xpack API endpoint's presence is correct for the type of distribution being tested.
-     */
-    public void test011PresenceOfXpack() throws Exception {
-        waitForElasticsearch(installation);
-        final int statusCode = Request.Get("http://localhost:9200/_xpack").execute().returnResponse().getStatusLine().getStatusCode();
-
-        if (distribution.isOSS()) {
-            assertThat(statusCode, greaterThanOrEqualTo(400));
-        } else {
-            assertThat(statusCode, equalTo(200));
-        }
-    }
-
     /**
      * Checks that no plugins are initially active.
      */

qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java

@@ -210,9 +210,7 @@ public class Archives {
             "elasticsearch-saml-metadata",
             "elasticsearch-setup-passwords",
             "elasticsearch-syskeygen",
-            "elasticsearch-users",
+            "elasticsearch-users"
-            "x-pack-env",
-            "x-pack-security-env"
         ).forEach(executable -> {
 
             assertThat(es.bin(executable), file(File, owner, owner, p755));

qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java

@@ -537,9 +537,7 @@ public class Docker {
             "elasticsearch-saml-metadata",
             "elasticsearch-setup-passwords",
             "elasticsearch-syskeygen",
-            "elasticsearch-users",
+            "elasticsearch-users"
-            "x-pack-env",
-            "x-pack-security-env"
         ).forEach(executable -> assertPermissionsAndOwnership(es.bin(executable), p755));
 
         Stream.of("role_mapping.yml", "roles.yml", "users", "users_roles")

qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java

@@ -238,9 +238,7 @@ public class Packages {
             "elasticsearch-saml-metadata",
             "elasticsearch-setup-passwords",
             "elasticsearch-syskeygen",
-            "elasticsearch-users",
+            "elasticsearch-users"
-            "x-pack-env",
-            "x-pack-security-env"
         ).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755)));
 
         Stream.of("users", "users_roles", "roles.yml", "role_mapping.yml", "log4j2.properties")

qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -41,8 +41,6 @@ import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManagerFactory;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.InetAddress;
-import java.net.Socket;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.KeyStore;
@@ -114,26 +112,6 @@ public class ServerUtils {
         return executor.execute(request).returnResponse();
     }
 
-    // polls every second for Elasticsearch to be running on 9200
-    private static void waitForXpack() {
-        int retries = 60;
-        while (retries > 0) {
-            retries -= 1;
-            try (Socket s = new Socket(InetAddress.getLoopbackAddress(), 9200)) {
-                return;
-            } catch (IOException e) {
-                // ignore, only want to establish a connection
-            }
-            try {
-                Thread.sleep(1000);
-            } catch (InterruptedException interrupted) {
-                Thread.currentThread().interrupt();
-                return;
-            }
-        }
-        throw new RuntimeException("Elasticsearch (with x-pack) did not start");
-    }
-
     public static void waitForElasticsearch(String status, String index, Installation installation, String username, String password)
         throws Exception {
 

qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/RemoteClustersIT.java

@@ -116,40 +116,4 @@ public class RemoteClustersIT extends AbstractMultiClusterRemoteTestCase {
         assertEquals(2L, cluster1Client().search(
             new SearchRequest("haproxynosn:test2"), RequestOptions.DEFAULT).getHits().getTotalHits().value);
     }
-
-    public void testHAProxyModeConnectionWithSNIToCluster1Works() throws IOException {
-        assumeThat("test is only supported if the distribution contains xpack", getDistribution(), equalTo("default"));
-
-        ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest().persistentSettings(Settings.builder()
-            .put("cluster.remote.haproxysni1.mode", "proxy")
-            .put("cluster.remote.haproxysni1.proxy_address", "haproxy:9600")
-            .put("cluster.remote.haproxysni1.server_name", "application1.example.com")
-            .build());
-        assertTrue(cluster2Client().cluster().putSettings(request, RequestOptions.DEFAULT).isAcknowledged());
-
-        RemoteConnectionInfo rci = cluster2Client().cluster().remoteInfo(new RemoteInfoRequest(), RequestOptions.DEFAULT).getInfos().get(0);
-        logger.info("Connection info: {}", rci);
-        assertTrue(rci.isConnected());
-
-        assertEquals(1L, cluster2Client().search(
-            new SearchRequest("haproxysni1:test1"), RequestOptions.DEFAULT).getHits().getTotalHits().value);
-    }
-
-    public void testHAProxyModeConnectionWithSNIToCluster2Works() throws IOException {
-        assumeThat("test is only supported if the distribution contains xpack", getDistribution(), equalTo("default"));
-
-        ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest().persistentSettings(Settings.builder()
-            .put("cluster.remote.haproxysni2.mode", "proxy")
-            .put("cluster.remote.haproxysni2.proxy_address", "haproxy:9600")
-            .put("cluster.remote.haproxysni2.server_name", "application2.example.com")
-            .build());
-        assertTrue(cluster1Client().cluster().putSettings(request, RequestOptions.DEFAULT).isAcknowledged());
-
-        RemoteConnectionInfo rci = cluster1Client().cluster().remoteInfo(new RemoteInfoRequest(), RequestOptions.DEFAULT).getInfos().get(0);
-        logger.info("Connection info: {}", rci);
-        assertTrue(rci.isConnected());
-
-        assertEquals(2L, cluster1Client().search(
-            new SearchRequest("haproxysni2:test2"), RequestOptions.DEFAULT).getHits().getTotalHits().value);
-    }
 }

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java

@@ -1,88 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.upgrades;
-
-import org.apache.http.util.EntityUtils;
-import org.junit.Before;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.rest.action.document.RestBulkAction;
-
-import java.io.IOException;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assume.assumeThat;
-
-/**
- * Basic tests for simple xpack functionality that are only run if the
- * cluster is the on the default distribution.
- */
-public class XPackIT extends AbstractRollingTestCase {
-    @Before
-    public void skipIfNotXPack() {
-        assumeThat("test is only supported if the distribution contains xpack",
-                System.getProperty("tests.distribution"), equalTo("default"));
-        assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
-                CLUSTER_TYPE, equalTo(ClusterType.UPGRADED));
-        /*
-         * *Mostly* we want this for when we're upgrading from pre-6.3's
-         * zip distribution which doesn't contain xpack to post 6.3's zip
-         * distribution which *does* contain xpack. But we'll also run it
-         * on all upgrades for completeness's sake.
-         */
-    }
-
-    /**
-     * Test creating a trial license and using it. This is interesting because
-     * our other tests test cover starting a new cluster with the default
-     * distribution and enabling the trial license but this test is the only
-     * one that can upgrade from the oss distribution to the default
-     * distribution with xpack and the create a trial license. We don't
-     * <strong>do</strong> a lot with the trial license because for the most
-     * part those things are tested elsewhere, off in xpack. But we do use the
-     * trial license a little bit to make sure that it works.
-     */
-    public void testTrialLicense() throws IOException {
-        Request startTrial = new Request("POST", "/_license/start_trial");
-        startTrial.addParameter("acknowledge", "true");
-        client().performRequest(startTrial);
-
-        String noJobs = EntityUtils.toString(
-            client().performRequest(new Request("GET", "/_ml/anomaly_detectors")).getEntity());
-        assertEquals("{\"count\":0,\"jobs\":[]}", noJobs);
-
-        Request createJob = new Request("PUT", "/_ml/anomaly_detectors/test_job");
-        createJob.setJsonEntity(
-                  "{\n"
-                + "  \"analysis_config\" : {\n"
-                + "    \"bucket_span\": \"10m\",\n"
-                + "    \"detectors\": [\n"
-                + "      {\n"
-                + "        \"function\": \"sum\",\n"
-                + "        \"field_name\": \"total\"\n"
-                + "      }\n"
-                + "    ]\n"
-                + "  },\n"
-                + "  \"data_description\": {\n"
-                + "    \"time_field\": \"timestamp\",\n"
-                + "    \"time_format\": \"epoch_ms\"\n"
-                + "  }\n"
-                + "}\n");
-        client().performRequest(createJob);
-    }
-}

server/build.gradle

@@ -175,7 +175,6 @@ tasks.named("testingConventions").configure {
 
 def generateModulesList = tasks.register("generateModulesList") {
     List<String> modules = project(':modules').subprojects.collect { it.name }
-    modules.add('x-pack')
     File modulesFile = new File(buildDir, 'generated-resources/modules.txt')
     processResources.from(modulesFile)
     inputs.property('modules', modules)

settings.gradle

@@ -114,7 +114,6 @@ addSubProjects('', new File(rootProject.projectDir, 'modules'))
 addSubProjects('', new File(rootProject.projectDir, 'plugins'))
 addSubProjects('', new File(rootProject.projectDir, 'qa'))
 addSubProjects('test', new File(rootProject.projectDir, 'test/external-modules'))
-addSubProjects('', new File(rootProject.projectDir, 'x-pack'))
 
 List startTasks = gradle.startParameter.taskNames