From 5b711c283dbc233f80de5e0adb26723c22d678c7 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 22 Sep 2017 02:26:32 -0700 Subject: [PATCH] Plugins: Add backcompat for sha1 checksums (#26748) With 6.0 rc1 we now publish sha512 checksums for official plugins. However, in order to ease the pain for plugin authors, this commit adds backcompat to still allow sha1 checksums. Also added tests for checksums. Closes #26746 --- .../plugins/InstallPluginCommand.java | 46 ++++++++-- .../plugins/InstallPluginCommandTests.java | 85 +++++++++++++++++-- 2 files changed, 118 insertions(+), 13 deletions(-) diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index 1ccb6f740dd..ec017cfffb5 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -58,6 +58,7 @@ import java.nio.file.attribute.PosixFileAttributeView; import java.nio.file.attribute.PosixFileAttributes; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; +import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -218,7 +219,7 @@ class InstallPluginCommand extends EnvironmentAwareCommand { if (OFFICIAL_PLUGINS.contains(pluginId)) { final String url = getElasticUrl(terminal, getStagingHash(), Version.CURRENT, pluginId, Platforms.PLATFORM_NAME); terminal.println("-> Downloading " + pluginId + " from elastic"); - return downloadZipAndChecksum(terminal, url, tmpDir); + return downloadZipAndChecksum(terminal, url, tmpDir, false); } // now try as maven coordinates, a valid URL would only have a colon and slash @@ -226,7 +227,7 @@ class InstallPluginCommand extends EnvironmentAwareCommand { if (coordinates.length == 3 && pluginId.contains("/") == false) { String mavenUrl = getMavenUrl(terminal, coordinates, Platforms.PLATFORM_NAME); terminal.println("-> Downloading " + pluginId + " from maven central"); - return downloadZipAndChecksum(terminal, mavenUrl, tmpDir); + return downloadZipAndChecksum(terminal, mavenUrl, tmpDir, true); } // fall back to plain old URL @@ -312,8 +313,9 @@ class InstallPluginCommand extends EnvironmentAwareCommand { } /** Downloads a zip from the url, into a temp file under the given temp dir. */ + // pkg private for tests @SuppressForbidden(reason = "We use getInputStream to download plugins") - private Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException { + Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException { terminal.println(VERBOSE, "Retrieving zip from " + urlString); URL url = new URL(urlString); Path zip = Files.createTempFile(tmpDir, null, ".zip"); @@ -361,13 +363,26 @@ class InstallPluginCommand extends EnvironmentAwareCommand { } } - /** Downloads a zip from the url, as well as a SHA1 checksum, and checks the checksum. */ + /** Downloads a zip from the url, as well as a SHA512 (or SHA1) checksum, and checks the checksum. */ // pkg private for tests @SuppressForbidden(reason = "We use openStream to download plugins") - Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception { + private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir, boolean allowSha1) throws Exception { Path zip = downloadZip(terminal, urlString, tmpDir); pathsToDeleteOnShutdown.add(zip); - URL checksumUrl = new URL(urlString + ".sha1"); + String checksumUrlString = urlString + ".sha512"; + URL checksumUrl = openUrl(checksumUrlString); + String digestAlgo = "SHA-512"; + if (checksumUrl == null && allowSha1) { + // fallback to sha1, until 7.0, but with warning + terminal.println("Warning: sha512 not found, falling back to sha1. This behavior is deprecated and will be removed in a " + + "future release. Please update the plugin to use a sha512 checksum."); + checksumUrlString = urlString + ".sha1"; + checksumUrl = openUrl(checksumUrlString); + digestAlgo = "SHA-1"; + } + if (checksumUrl == null) { + throw new UserException(ExitCodes.IO_ERROR, "Plugin checksum missing: " + checksumUrlString); + } final String expectedChecksum; try (InputStream in = checksumUrl.openStream()) { BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); @@ -378,15 +393,30 @@ class InstallPluginCommand extends EnvironmentAwareCommand { } byte[] zipbytes = Files.readAllBytes(zip); - String gotChecksum = MessageDigests.toHexString(MessageDigests.sha1().digest(zipbytes)); + String gotChecksum = MessageDigests.toHexString(MessageDigest.getInstance(digestAlgo).digest(zipbytes)); if (expectedChecksum.equals(gotChecksum) == false) { throw new UserException(ExitCodes.IO_ERROR, - "SHA1 mismatch, expected " + expectedChecksum + " but got " + gotChecksum); + digestAlgo + " mismatch, expected " + expectedChecksum + " but got " + gotChecksum); } return zip; } + /** + * Creates a URL and opens a connection. + * + * If the URL returns a 404, {@code null} is returned, otherwise the open URL opject is returned. + */ + // pkg private for tests + URL openUrl(String urlString) throws Exception { + URL checksumUrl = new URL(urlString); + HttpURLConnection connection = (HttpURLConnection)checksumUrl.openConnection(); + if (connection.getResponseCode() == 404) { + return null; + } + return checksumUrl; + } + private Path unzip(Path zip, Path pluginsDir) throws IOException, UserException { // unzip plugin to a staging temp dir diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index d8166a348f6..103baccd2d4 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -24,11 +24,13 @@ import com.google.common.jimfs.Configuration; import com.google.common.jimfs.Jimfs; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; +import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.Terminal; import org.elasticsearch.cli.UserException; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.io.PathUtilsForTesting; @@ -62,7 +64,7 @@ import java.nio.file.attribute.PosixFileAttributeView; import java.nio.file.attribute.PosixFileAttributes; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.UserPrincipal; -import java.security.KeyStore; +import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -751,19 +753,33 @@ public class InstallPluginCommandTests extends ESTestCase { skipJarHellCommand.execute(terminal, pluginZip, isBatch, env.v2()); } - public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception { + public MockTerminal assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash, + String shaExtension, Function shaCalculator) throws Exception { Tuple env = createEnv(fs, temp); Path pluginDir = createPluginDir(temp); Path pluginZip = createPlugin(name, pluginDir, false); InstallPluginCommand command = new InstallPluginCommand() { @Override - Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception { + Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException { assertEquals(url, urlString); Path downloadedPath = tmpDir.resolve("downloaded.zip"); Files.copy(pluginZip, downloadedPath); return downloadedPath; } @Override + URL openUrl(String urlString) throws Exception { + String expectedUrl = url + shaExtension; + if (expectedUrl.equals(urlString)) { + // calc sha an return file URL to it + Path shaFile = temp.apply("shas").resolve("downloaded.zip" + shaExtension); + byte[] zipbytes = Files.readAllBytes(pluginZip); + String checksum = shaCalculator.apply(zipbytes); + Files.write(shaFile, checksum.getBytes(StandardCharsets.UTF_8)); + return shaFile.toUri().toURL(); + } + return null; + } + @Override boolean urlExists(Terminal terminal, String urlString) throws IOException { return urlString.equals(url); } @@ -776,8 +792,15 @@ public class InstallPluginCommandTests extends ESTestCase { // no jarhell check } }; - installPlugin(pluginId, env.v1(), command); + MockTerminal terminal = installPlugin(pluginId, env.v1(), command); assertPlugin(name, pluginDir, env.v2()); + return terminal; + } + + public void assertInstallPluginFromUrl(String pluginId, String name, String url, String stagingHash) throws Exception { + MessageDigest digest = MessageDigest.getInstance("SHA-512"); + assertInstallPluginFromUrl(pluginId, name, url, stagingHash, ".sha512", + bytes -> MessageDigests.toHexString(digest.digest(bytes))); } public void testOfficalPlugin() throws Exception { @@ -813,7 +836,59 @@ public class InstallPluginCommandTests extends ESTestCase { assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null); } - // TODO: test checksum (need maven/official below) + public void testMavenSha1Backcompat() throws Exception { + String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + MockTerminal terminal = assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, + ".sha1", bytes -> MessageDigests.toHexString(digest.digest(bytes))); + assertTrue(terminal.getOutput(), terminal.getOutput().contains("sha512 not found, falling back to sha1")); + } + + public void testOfficialShaMissing() throws Exception { + String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha1", + bytes -> MessageDigests.toHexString(digest.digest(bytes)))); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage()); + } + + public void testMavenShaMissing() throws Exception { + String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, ".dne", bytes -> null)); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage()); + } + + public void testInvalidShaFile() throws Exception { + String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; + MessageDigest digest = MessageDigest.getInstance("SHA-512"); + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512", + bytes -> MessageDigests.toHexString(digest.digest(bytes)) + "\nfoobar")); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file")); + } + + public void testSha512Mismatch() throws Exception { + String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Version.CURRENT + ".zip"; + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, ".sha512", + bytes -> "foobar")); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertTrue(e.getMessage(), e.getMessage().contains("SHA-512 mismatch, expected foobar")); + } + + public void testSha1Mismatch() throws Exception { + String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; + UserException e = expectThrows(UserException.class, () -> + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, + ".sha1", bytes -> "foobar")); + assertEquals(ExitCodes.IO_ERROR, e.exitCode); + assertTrue(e.getMessage(), e.getMessage().contains("SHA-1 mismatch, expected foobar")); + } public void testKeystoreNotRequired() throws Exception { Tuple env = createEnv(fs, temp);