honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable installing plugins from snapshots.elastic.co (#30765) 

We post snapshot builds to snapshots.elastic.co yet the official plugin
installer will not let you install such plugins without manually
downloading them and installing them from a file URL. This commit adds
the ability for the plugin installer to use snapshots.elastic.co for
installing official plugins if a es.plugins.staging is set and the
current build is also a snapshot build. Otherwise, we continue to use
staging.elastic.co if the current build is a release build and
es.plugins.staging is set and, of course, use the release artifacts at
artifacts.elastic.co for release builds with es.plugins.staging unset.
This commit is contained in:
Jason Tedor 2018-05-22 01:09:31 -04:00 committed by GitHub
parent 8b7ff69479
commit 2844bcbcc8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 89 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
View File

@ -30,7 +30,6 @@ import org.elasticsearch.cli.EnvironmentAwareCommand;
import org.elasticsearch.cli.ExitCodes;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.cli.UserException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.hash.MessageDigests;
@ -240,7 +239,7 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
/** Downloads the plugin and returns the file it was downloaded to. */
private Path download(Terminal terminal, String pluginId, Path tmpDir) throws Exception {
if (OFFICIAL_PLUGINS.contains(pluginId)) {
final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, pluginId, Platforms.PLATFORM_NAME);
final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, isSnapshot(), pluginId, Platforms.PLATFORM_NAME);
terminal.println("-> Downloading " + pluginId + " from elastic");
return downloadZipAndChecksum(terminal, url, tmpDir, false);
}
@ -272,22 +271,43 @@ class InstallPluginCommand extends EnvironmentAwareCommand {
return System.getProperty(PROPERTY_STAGING_ID);
}
boolean isSnapshot() {
return Build.CURRENT.isSnapshot();
}
/** Returns the url for an official elasticsearch plugin. */
private String getElasticUrl(Terminal terminal, String stagingHash, Version version,
String pluginId, String platform) throws IOException {
private String getElasticUrl(
final Terminal terminal,
final String stagingHash,
final Version version,
final boolean isSnapshot,
final String pluginId,
final String platform) throws IOException, UserException {
final String baseUrl;
if (stagingHash != null) {
baseUrl = String.format(Locale.ROOT,
"https://staging.elastic.co/%s-%s/downloads/elasticsearch-plugins/%s", version, stagingHash, pluginId);
} else {
baseUrl = String.format(Locale.ROOT,
"https://artifacts.elastic.co/downloads/elasticsearch-plugins/%s", pluginId);
if (isSnapshot && stagingHash == null) {
throw new UserException(
ExitCodes.CONFIG, "attempted to install release build of official plugin on snapshot build of Elasticsearch");
}
final String platformUrl = String.format(Locale.ROOT, "%s/%s-%s-%s.zip", baseUrl, pluginId, platform, version);
if (stagingHash != null) {
if (isSnapshot) {
baseUrl = nonReleaseUrl("snapshots", version, stagingHash, pluginId);
} else {
baseUrl = nonReleaseUrl("staging", version, stagingHash, pluginId);
}
} else {
baseUrl = String.format(Locale.ROOT, "https://artifacts.elastic.co/downloads/elasticsearch-plugins/%s", pluginId);
}
final String platformUrl =
String.format(Locale.ROOT, "%s/%s-%s-%s.zip", baseUrl, pluginId, platform, Version.displayVersion(version, isSnapshot));
if (urlExists(terminal, platformUrl)) {
return platformUrl;
}
return String.format(Locale.ROOT, "%s/%s-%s.zip", baseUrl, pluginId, version);
return String.format(Locale.ROOT, "%s/%s-%s.zip", baseUrl, pluginId, Version.displayVersion(version, isSnapshot));
}
private String nonReleaseUrl(final String hostname, final Version version, final String stagingHash, final String pluginId) {
return String.format(
Locale.ROOT, "https://%s.elastic.co/%s-%s/downloads/elasticsearch-plugins/%s", hostname, version, stagingHash, pluginId);
}
/** Returns the url for an elasticsearch plugin in maven. */

71
distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
View File

@ -800,7 +800,7 @@ public class InstallPluginCommandTests extends ESTestCase {
skipJarHellCommand.execute(terminal, pluginZip, isBatch, env.v2());
}
void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash,
void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash, boolean isSnapshot,
String shaExtension, Function<byte[], String> shaCalculator) throws Exception {
Tuple<Path, Environment> env = createEnv(fs, temp);
Path pluginDir = createPluginDir(temp);
@ -834,6 +834,12 @@ public class InstallPluginCommandTests extends ESTestCase {
String getStagingHash() {
return stagingHash;
}
@Override
boolean isSnapshot() {
return isSnapshot;
}
@Override
void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Path modulesDir) throws Exception {
// no jarhell check
@ -843,48 +849,82 @@ public class InstallPluginCommandTests extends ESTestCase {
assertPlugin(name, pluginDir, env.v2());
}
public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception {
public void assertInstallPluginFromUrl(
final String pluginId, final String name, final String url, final String stagingHash, boolean isSnapshot) throws Exception {
MessageDigest digest = MessageDigest.getInstance("SHA-512");
assertInstallPluginFromUrl(pluginId, name, url, stagingHash, ".sha512", checksumAndFilename(digest, url));
assertInstallPluginFromUrl(pluginId, name, url, stagingHash, isSnapshot, ".sha512", checksumAndFilename(digest, url));
}
public void testOfficalPlugin() throws Exception {
String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null);
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false);
}
public void testOfficialPluginSnapshot() throws Exception {
String url = String.format(
Locale.ROOT,
"https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip",
Version.CURRENT,
Version.displayVersion(Version.CURRENT, true));
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true);
}
public void testInstallReleaseBuildOfPluginOnSnapshotBuild() {
String url = String.format(
Locale.ROOT,
"https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip",
Version.CURRENT,
Version.displayVersion(Version.CURRENT, true));
// attemping to install a release build of a plugin (no staging ID) on a snapshot build should throw a user exception
final UserException e =
expectThrows(UserException.class, () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, true));
assertThat(e.exitCode, equalTo(ExitCodes.CONFIG));
assertThat(
e, hasToString(containsString("attempted to install release build of official plugin on snapshot build of Elasticsearch")));
}
public void testOfficalPluginStaging() throws Exception {
String url = "https://staging.elastic.co/" + Version.CURRENT + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-"
+ Version.CURRENT + ".zip";
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123");
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false);
}
public void testOfficalPlatformPlugin() throws Exception {
String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Platforms.PLATFORM_NAME +
"-" + Version.CURRENT + ".zip";
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null);
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false);
}
public void testOfficialPlatformPluginSnapshot() throws Exception {
String url = String.format(
Locale.ROOT,
"https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s-%s.zip",
Version.CURRENT,
Platforms.PLATFORM_NAME,
Version.displayVersion(Version.CURRENT, true));
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true);
}
public void testOfficalPlatformPluginStaging() throws Exception {
String url = "https://staging.elastic.co/" + Version.CURRENT + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-"
+ Platforms.PLATFORM_NAME + "-"+ Version.CURRENT + ".zip";
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123");
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false);
}
public void testMavenPlugin() throws Exception {
String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null);
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false);
}
public void testMavenPlatformPlugin() throws Exception {
String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-" + Platforms.PLATFORM_NAME + "-1.0.0.zip";
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null);
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false);
}
public void testMavenSha1Backcompat() throws Exception {
String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
MessageDigest digest = MessageDigest.getInstance("SHA-1");
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".sha1", checksum(digest));
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".sha1", checksum(digest));
assertTrue(terminal.getOutput(), terminal.getOutput().contains("sha512 not found, falling back to sha1"));
}
@ -892,7 +932,7 @@ public class InstallPluginCommandTests extends ESTestCase {
String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
MessageDigest digest = MessageDigest.getInstance("SHA-1");
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha1", checksum(digest)));
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false, ".sha1", checksum(digest)));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage());
}
@ -900,7 +940,7 @@ public class InstallPluginCommandTests extends ESTestCase {
public void testMavenShaMissing() throws Exception {
String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".dne", bytes -> null));
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".dne", bytes -> null));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage());
}
@ -909,7 +949,7 @@ public class InstallPluginCommandTests extends ESTestCase {
String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip";
MessageDigest digest = MessageDigest.getInstance("SHA-512");
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512", checksum(digest)));
assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false, ".sha512", checksum(digest)));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file"));
}
@ -923,6 +963,7 @@ public class InstallPluginCommandTests extends ESTestCase {
"analysis-icu",
url,
null,
false,
".sha512",
checksumAndString(digest, " repository-s3-" + Version.CURRENT + ".zip")));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
@ -938,6 +979,7 @@ public class InstallPluginCommandTests extends ESTestCase {
"analysis-icu",
url,
null,
false,
".sha512",
checksumAndString(digest, " analysis-icu-" + Version.CURRENT + ".zip\nfoobar")));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
@ -952,6 +994,7 @@ public class InstallPluginCommandTests extends ESTestCase {
"analysis-icu",
url,
null,
false,
".sha512",
bytes -> "foobar analysis-icu-" + Version.CURRENT + ".zip"));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
@ -961,7 +1004,7 @@ public class InstallPluginCommandTests extends ESTestCase {
public void testSha1Mismatch() throws Exception {
String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip";
UserException e = expectThrows(UserException.class, () ->
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".sha1", bytes -> "foobar"));
assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".sha1", bytes -> "foobar"));
assertEquals(ExitCodes.IO_ERROR, e.exitCode);
assertTrue(e.getMessage(), e.getMessage().contains("SHA-1 mismatch, expected foobar"));
}