From cdd06d40d2ef012521184b5ddcc6e3dad2cf3eba Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 20 Jul 2019 15:18:16 -0700 Subject: [PATCH] Do not checksum all bytes at once in plugin install (#44649) Today when checksumming a plugin zip during plugin install, we read all of the bytes of the zip into memory at once. When trying to run the plugin installer on a small heap (say, 64 MiB), this can lead to the plugin installer running out of memory when checksumming large plugins. This commit addresses this by reading the plugin bytes in 8 KiB chunks, thus using a constant amount of memory independent of the size of the plugin. --- .../plugins/InstallPluginCommand.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index 7797dc39bda..073c69bb9ac 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -506,17 +506,26 @@ class InstallPluginCommand extends EnvironmentAwareCommand { } } - try { - final byte[] zipBytes = Files.readAllBytes(zip); - final String actualChecksum = MessageDigests.toHexString(MessageDigest.getInstance(digestAlgo).digest(zipBytes)); - if (expectedChecksum.equals(actualChecksum) == false) { - throw new UserException( + // read the bytes of the plugin zip in chunks to avoid out of memory errors + try (InputStream zis = Files.newInputStream(zip)) { + try { + final MessageDigest digest = MessageDigest.getInstance(digestAlgo); + final byte[] bytes = new byte[8192]; + int read; + while ((read = zis.read(bytes)) != -1) { + assert read > 0 : read; + digest.update(bytes, 0, read); + } + final String actualChecksum = MessageDigests.toHexString(digest.digest()); + if (expectedChecksum.equals(actualChecksum) == false) { + throw new UserException( ExitCodes.IO_ERROR, digestAlgo + " mismatch, expected " + expectedChecksum + " but got " + actualChecksum); + } + } catch (final NoSuchAlgorithmException e) { + // this should never happen as we are using SHA-1 and SHA-512 here + throw new AssertionError(e); } - } catch (final NoSuchAlgorithmException e) { - // this should never happen as we are using SHA-1 and SHA-512 here - throw new AssertionError(e); } if (officialPlugin) {