From bf97679854b776878e3a98485c30adfd52cf748d Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 12 May 2020 11:10:05 +0100 Subject: [PATCH] Push arch-specific logic from Gradle into Docker (#56503) Docker informed us that for official multi-arch Docker builds, there needs to be a single Dockerfile and build context that can be used for each supported architecture. Therefore, rework the build to move the relevant architecture logic into the Dockerfile, and merge the aarch64 / x64 docker context builds. --- distribution/docker/build.gradle | 57 ++++++----- .../docker-aarch64-build-context/build.gradle | 11 --- .../docker/docker-build-context/build.gradle | 4 +- .../build.gradle | 11 --- .../oss-docker-build-context/build.gradle | 4 +- distribution/docker/src/docker/Dockerfile | 94 ++++++++++--------- 6 files changed, 93 insertions(+), 88 deletions(-) delete mode 100644 distribution/docker/docker-aarch64-build-context/build.gradle delete mode 100644 distribution/docker/oss-docker-aarch64-build-context/build.gradle diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index c9ecceac110..e4dd5e1ba30 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -28,32 +28,47 @@ dependencies { } ext.expansions = { architecture, oss, local -> - String base_image = null - String tini_arch = null - String classifier = null - switch (architecture) { - case "aarch64": - base_image = "arm64v8/centos:7" - tini_arch = "arm64" - classifier = "linux-aarch64" - break; - case "x64": - base_image = "amd64/centos:7" - tini_arch = "amd64" - classifier = "linux-x86_64" - break; - default: - throw new IllegalArgumentException("unrecongized architecture [" + architecture + "], must be one of (aarch64|x64)") + String classifier + if (local) { + switch (architecture) { + case "aarch64": + classifier = "linux-aarch64" + break + case "x64": + classifier = "linux-x86_64" + break + default: + throw new IllegalArgumentException("Unrecognized architecture [" + architecture + "], must be one of (aarch64|x64)") + } + } else { + /* When sourcing the Elasticsearch build remotely, the same Dockerfile needs + * to be able to fetch the artifact for any supported platform. We can't make + * the decision here. Bash will interpolate the `arch` command for us. */ + classifier = "linux-\$(arch)" } - final String elasticsearch = oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}-${classifier}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}-${classifier}.tar.gz" + + final String elasticsearch = "elasticsearch-${oss ? 'oss-' : ''}${VersionProperties.elasticsearch}-${classifier}.tar.gz" + + /* Both the following Dockerfile commands put the resulting artifact at + * the same location, regardless of classifier, so that the commands that + * follow in the Dockerfile don't have to know about the runtime + * architecture. */ + String sourceElasticsearch + if (local) { + sourceElasticsearch = "COPY $elasticsearch /opt/elasticsearch.tar.gz" + } else { + sourceElasticsearch = """ +RUN curl --retry 8 -S -L \\ + --output /opt/elasticsearch.tar.gz \\ + https://artifacts.elastic.co/downloads/elasticsearch/$elasticsearch +""" + } + return [ - 'base_image' : base_image, 'build_date' : BuildParams.buildDate, - 'elasticsearch' : elasticsearch, 'git_revision' : BuildParams.gitRevision, 'license' : oss ? 'Apache-2.0' : 'Elastic-License', - 'source_elasticsearch': local ? "COPY $elasticsearch /opt/" : "RUN cd /opt && curl --retry 8 -s -L -O https://artifacts.elastic.co/downloads/elasticsearch/${elasticsearch} && cd -", - 'tini_arch' : tini_arch, + 'source_elasticsearch': sourceElasticsearch, 'version' : VersionProperties.elasticsearch ] } diff --git a/distribution/docker/docker-aarch64-build-context/build.gradle b/distribution/docker/docker-aarch64-build-context/build.gradle deleted file mode 100644 index b93a403bf64..00000000000 --- a/distribution/docker/docker-aarch64-build-context/build.gradle +++ /dev/null @@ -1,11 +0,0 @@ -apply plugin: 'base' - -task buildDockerBuildContext(type: Tar) { - extension = 'tar.gz' - compression = Compression.GZIP - archiveClassifier = "docker-build-context-aarch64" - archiveBaseName = "elasticsearch" - with dockerBuildContext("aarch64", false, false) -} - -assemble.dependsOn buildDockerBuildContext diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle index 2dd28329d7b..3bd3a9059c7 100644 --- a/distribution/docker/docker-build-context/build.gradle +++ b/distribution/docker/docker-build-context/build.gradle @@ -5,7 +5,9 @@ task buildDockerBuildContext(type: Tar) { compression = Compression.GZIP archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch" - with dockerBuildContext("x64", false, false) + // Non-local builds don't need to specify an architecture. + // Make this explicit via the string value. + with dockerBuildContext("", false, false) } assemble.dependsOn buildDockerBuildContext diff --git a/distribution/docker/oss-docker-aarch64-build-context/build.gradle b/distribution/docker/oss-docker-aarch64-build-context/build.gradle deleted file mode 100644 index cb6d89d55ac..00000000000 --- a/distribution/docker/oss-docker-aarch64-build-context/build.gradle +++ /dev/null @@ -1,11 +0,0 @@ -apply plugin: 'base' - -task buildOssDockerBuildContext(type: Tar) { - extension = 'tar.gz' - compression = Compression.GZIP - archiveClassifier = "docker-build-context-aarch64" - archiveBaseName = "elasticsearch-oss" - with dockerBuildContext("aarch64", true, false) -} - -assemble.dependsOn buildOssDockerBuildContext diff --git a/distribution/docker/oss-docker-build-context/build.gradle b/distribution/docker/oss-docker-build-context/build.gradle index 0a29c2a2b72..904828d7651 100644 --- a/distribution/docker/oss-docker-build-context/build.gradle +++ b/distribution/docker/oss-docker-build-context/build.gradle @@ -5,7 +5,9 @@ task buildOssDockerBuildContext(type: Tar) { compression = Compression.GZIP archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch-oss" - with dockerBuildContext("x64", true, false) + // Non-local builds don't need to specify an architecture. + // Make this explicit via the string value. + with dockerBuildContext("", true, false) } assemble.dependsOn buildOssDockerBuildContext diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index e8babab244a..7d2cc2cd2d9 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -3,45 +3,53 @@ # # Beginning of multi stage Dockerfile ################################################################################ +<% /* + This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes + have to be escaped in order for them to appear in the final Dockerfile. You + can also comment out blocks, like this one. See: + https://docs.groovy-lang.org/latest/html/api/groovy/text/SimpleTemplateEngine.html +*/ %> ################################################################################ # Build stage 0 `builder`: # Extract elasticsearch artifact -# Install required plugins # Set gid=0 and make group perms==owner perms ################################################################################ -FROM ${base_image} AS builder - -RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \ - yum install --setopt=tsflags=nodocs -y wget gzip shadow-utils tar && \ - yum clean all && exit_code=0 && break || exit_code=\$? && echo "yum error: retry \$iter in 10s" && sleep 10; done; \ - (exit \$exit_code) +FROM centos:7 AS builder # `tini` is a tiny but valid init for containers. This is used to cleanly # control how ES and any child processes are shut down. # # The tini GitHub page gives instructions for verifying the binary using # gpg, but the keyservers are slow to return the key and this can fail the -# build. Instead, we check the binary against a checksum that they provide. -RUN wget --no-cookies --quiet https://github.com/krallin/tini/releases/download/v0.19.0/tini-${tini_arch} \ - && wget --no-cookies --quiet https://github.com/krallin/tini/releases/download/v0.19.0/tini-${tini_arch}.sha256sum \ - && sha256sum -c tini-${tini_arch}.sha256sum \ - && mv tini-${tini_arch} /tini \ - && chmod +x /tini +# build. Instead, we check the binary against the published checksum. +RUN set -eux ; \\ + \\ + tini_bin="" ; \\ + case "\$(arch)" in \\ + aarch64) tini_bin='tini-arm64' ;; \\ + x86_64) tini_bin='tini-amd64' ;; \\ + *) echo >&2 ; echo >&2 "Unsupported architecture \$(arch)" ; echo >&2 ; exit 1 ;; \\ + esac ; \\ + curl --retry 8 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin} ; \\ + curl --retry 8 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\ + sha256sum -c \${tini_bin}.sha256sum ; \\ + rm \${tini_bin}.sha256sum ; \\ + mv \${tini_bin} /tini ; \\ + chmod +x /tini ENV PATH /usr/share/elasticsearch/bin:\$PATH -RUN groupadd -g 1000 elasticsearch && \ +RUN groupadd -g 1000 elasticsearch && \\ adduser -u 1000 -g 1000 -d /usr/share/elasticsearch elasticsearch WORKDIR /usr/share/elasticsearch ${source_elasticsearch} -RUN tar zxf /opt/${elasticsearch} --strip-components=1 -RUN grep ES_DISTRIBUTION_TYPE=tar /usr/share/elasticsearch/bin/elasticsearch-env \ - && sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env +RUN tar zxf /opt/elasticsearch.tar.gz --strip-components=1 +RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env RUN mkdir -p config config/jvm.options.d data logs RUN chmod 0775 config config/jvm.options.d data logs COPY config/elasticsearch.yml config/log4j2.properties config/ @@ -53,20 +61,20 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties # Add entrypoint ################################################################################ -FROM ${base_image} +FROM centos:7 ENV ELASTIC_CONTAINER true COPY --from=builder /tini /tini -RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \ - yum install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \ - yum clean all && exit_code=0 && break || exit_code=\$? && echo "yum error: retry \$iter in 10s" && sleep 10; done; \ +RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \\ + yum install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \\ + yum clean all && exit_code=0 && break || exit_code=\$? && echo "yum error: retry \$iter in 10s" && sleep 10; done; \\ (exit \$exit_code) -RUN groupadd -g 1000 elasticsearch && \ - adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \ - chmod 0775 /usr/share/elasticsearch && \ +RUN groupadd -g 1000 elasticsearch && \\ + adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\ + chmod 0775 /usr/share/elasticsearch && \\ chgrp 0 /usr/share/elasticsearch WORKDIR /usr/share/elasticsearch @@ -81,7 +89,7 @@ ENV PATH /usr/share/elasticsearch/bin:\$PATH COPY bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh -RUN chmod g=u /etc/passwd && \ +RUN chmod g=u /etc/passwd && \\ chmod 0775 /usr/local/bin/docker-entrypoint.sh # Ensure that there are no files with setuid or setgid, in order to mitigate "stackclash" attacks. @@ -89,24 +97,24 @@ RUN find / -xdev -perm -4000 -exec chmod ug-s {} + EXPOSE 9200 9300 -LABEL org.label-schema.build-date="${build_date}" \ - org.label-schema.license="${license}" \ - org.label-schema.name="Elasticsearch" \ - org.label-schema.schema-version="1.0" \ - org.label-schema.url="https://www.elastic.co/products/elasticsearch" \ - org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \ - org.label-schema.vcs-ref="${git_revision}" \ - org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \ - org.label-schema.vendor="Elastic" \ - org.label-schema.version="${version}" \ - org.opencontainers.image.created="${build_date}" \ - org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \ - org.opencontainers.image.licenses="${license}" \ - org.opencontainers.image.revision="${git_revision}" \ - org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \ - org.opencontainers.image.title="Elasticsearch" \ - org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \ - org.opencontainers.image.vendor="Elastic" \ +LABEL org.label-schema.build-date="${build_date}" \\ + org.label-schema.license="${license}" \\ + org.label-schema.name="Elasticsearch" \\ + org.label-schema.schema-version="1.0" \\ + org.label-schema.url="https://www.elastic.co/products/elasticsearch" \\ + org.label-schema.usage="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\ + org.label-schema.vcs-ref="${git_revision}" \\ + org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \\ + org.label-schema.vendor="Elastic" \\ + org.label-schema.version="${version}" \\ + org.opencontainers.image.created="${build_date}" \\ + org.opencontainers.image.documentation="https://www.elastic.co/guide/en/elasticsearch/reference/index.html" \\ + org.opencontainers.image.licenses="${license}" \\ + org.opencontainers.image.revision="${git_revision}" \\ + org.opencontainers.image.source="https://github.com/elastic/elasticsearch" \\ + org.opencontainers.image.title="Elasticsearch" \\ + org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\ + org.opencontainers.image.vendor="Elastic" \\ org.opencontainers.image.version="${version}" ENTRYPOINT ["/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]