Introduce Docker images build (#36246)
This commit introduces the building of the Docker images as bonafide packaging formats alongside our existing archive and packaging distributions. This build is migrated from a dedicated repository, and converted to Gradle in the process.
This commit is contained in:
Jason Tedor
GitHub
parent
3e04a90e99
commit
11dd412ec6
|
|
@ -51,6 +51,7 @@ import org.gradle.api.tasks.compile.JavaCompile
|
|||
import org.gradle.api.tasks.javadoc.Javadoc
|
||||
import org.gradle.internal.jvm.Jvm
|
||||
import org.gradle.process.ExecResult
|
||||
import org.gradle.process.ExecSpec
|
||||
import org.gradle.util.GradleVersion
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
|
|
@ -232,6 +233,95 @@ class BuildPlugin implements Plugin<Project> {
|
|||
project.ext.java9Home = project.rootProject.ext.java9Home
|
||||
}
|
||||
|
||||
static void requireDocker(final Task task) {
|
||||
final Project rootProject = task.project.rootProject
|
||||
if (rootProject.hasProperty('requiresDocker') == false) {
|
||||
/*
|
||||
* This is our first time encountering a task that requires Docker. We will add an extension that will let us track the tasks
|
||||
* that register as requiring Docker. We will add a delayed execution that when the task graph is ready if any such tasks are
|
||||
* in the task graph, then we check two things:
|
||||
* - the Docker binary is available
|
||||
* - we can execute a Docker command that requires privileges
|
||||
*
|
||||
* If either of these fail, we fail the build.
|
||||
*/
|
||||
final boolean buildDocker
|
||||
final String buildDockerProperty = System.getProperty("build.docker")
|
||||
if (buildDockerProperty == null || buildDockerProperty == "true") {
|
||||
buildDocker = true
|
||||
} else if (buildDockerProperty == "false") {
|
||||
buildDocker = false
|
||||
} else {
|
||||
throw new IllegalArgumentException(
|
||||
"expected build.docker to be unset or one of \"true\" or \"false\" but was [" + buildDockerProperty + "]")
|
||||
}
|
||||
rootProject.rootProject.ext.buildDocker = buildDocker
|
||||
rootProject.rootProject.ext.requiresDocker = []
|
||||
rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
|
||||
// check if the Docker binary exists and record its path
|
||||
final List<String> maybeDockerBinaries = ['/usr/bin/docker2', '/usr/local/bin/docker2']
|
||||
final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() }
|
||||
|
||||
int exitCode
|
||||
String dockerErrorOutput
|
||||
if (dockerBinary == null) {
|
||||
exitCode = -1
|
||||
dockerErrorOutput = null
|
||||
} else {
|
||||
// the Docker binary executes, check that we can execute a privileged command
|
||||
final ByteArrayOutputStream output = new ByteArrayOutputStream()
|
||||
final ExecResult result = LoggedExec.exec(rootProject, { ExecSpec it ->
|
||||
it.commandLine dockerBinary, "images"
|
||||
it.errorOutput = output
|
||||
it.ignoreExitValue = true
|
||||
})
|
||||
if (result.exitValue == 0) {
|
||||
return
|
||||
}
|
||||
exitCode = result.exitValue
|
||||
dockerErrorOutput = output.toString()
|
||||
}
|
||||
final List<String> tasks =
|
||||
((List<Task>)rootProject.requiresDocker).findAll { taskGraph.hasTask(it) }.collect { " ${it.path}".toString()}
|
||||
if (tasks.isEmpty() == false) {
|
||||
/*
|
||||
* There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not
|
||||
* exist or because execution of a privileged Docker command failed.
|
||||
*/
|
||||
String message
|
||||
if (dockerBinary == null) {
|
||||
message = String.format(
|
||||
Locale.ROOT,
|
||||
"Docker (checked [%s]) is required to run the following task%s: \n%s",
|
||||
maybeDockerBinaries.join(","),
|
||||
tasks.size() > 1 ? "s" : "",
|
||||
tasks.join('\n'))
|
||||
} else {
|
||||
assert exitCode > 0 && dockerErrorOutput != null
|
||||
message = String.format(
|
||||
Locale.ROOT,
|
||||
"a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" +
|
||||
"the problem is that Docker exited with exit code [%d] with standard error output [%s]",
|
||||
dockerBinary,
|
||||
tasks.size() > 1 ? "s" : "",
|
||||
tasks.join('\n'),
|
||||
exitCode,
|
||||
dockerErrorOutput.trim())
|
||||
}
|
||||
throw new GradleException(
|
||||
message + "\nyou can address this by attending to the reported issue, "
|
||||
+ "removing the offending tasks from being executed, "
|
||||
+ "or by passing -Dbuild.docker=false")
|
||||
}
|
||||
}
|
||||
}
|
||||
if (rootProject.buildDocker) {
|
||||
rootProject.requiresDocker.add(task)
|
||||
} else {
|
||||
task.enabled = false
|
||||
}
|
||||
}
|
||||
|
||||
private static String findCompilerJavaHome() {
|
||||
String compilerJavaHome = System.getenv('JAVA_HOME')
|
||||
final String compilerJavaProperty = System.getProperty('compiler.java')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,106 @@
|
|||
import org.elasticsearch.gradle.BuildPlugin
|
||||
import org.elasticsearch.gradle.LoggedExec
|
||||
import org.elasticsearch.gradle.MavenFilteringHack
|
||||
import org.elasticsearch.gradle.VersionProperties
|
||||
|
||||
apply plugin: 'base'
|
||||
|
||||
configurations {
|
||||
dockerPlugins
|
||||
dockerSource
|
||||
ossDockerSource
|
||||
}
|
||||
|
||||
dependencies {
|
||||
dockerPlugins project(path: ":plugins:ingest-geoip", configuration: 'zip')
|
||||
dockerPlugins project(path: ":plugins:ingest-user-agent", configuration: 'zip')
|
||||
dockerSource project(path: ":distribution:archives:tar")
|
||||
ossDockerSource project(path: ":distribution:archives:oss-tar")
|
||||
}
|
||||
|
||||
ext.expansions = { oss ->
|
||||
return [
|
||||
'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz",
|
||||
'jdkUrl' : 'https://download.java.net/java/GA/jdk11/13/GPL/openjdk-11.0.1_linux-x64_bin.tar.gz',
|
||||
'jdkVersion' : '11.0.1',
|
||||
'license': oss ? 'Apache-2.0' : 'Elastic License',
|
||||
'ingest-geoip' : "ingest-geoip-${VersionProperties.elasticsearch}.zip",
|
||||
'ingest-user-agent' : "ingest-user-agent-${VersionProperties.elasticsearch}.zip",
|
||||
'version' : VersionProperties.elasticsearch
|
||||
]
|
||||
}
|
||||
|
||||
private static String files(final boolean oss) {
|
||||
return "build/${ oss ? 'oss-' : ''}docker"
|
||||
}
|
||||
|
||||
private static String taskName(final String prefix, final boolean oss, final String suffix) {
|
||||
return "${prefix}${oss ? 'Oss' : ''}${suffix}"
|
||||
}
|
||||
|
||||
void addCopyDockerContextTask(final boolean oss) {
|
||||
task(taskName("copy", oss, "DockerContext"), type: Sync) {
|
||||
into files(oss)
|
||||
|
||||
into('bin') {
|
||||
from 'src/docker/bin'
|
||||
}
|
||||
|
||||
into('config') {
|
||||
from 'src/docker/config'
|
||||
}
|
||||
|
||||
if (oss) {
|
||||
from configurations.ossDockerSource
|
||||
} else {
|
||||
from configurations.dockerSource
|
||||
}
|
||||
|
||||
from configurations.dockerPlugins
|
||||
}
|
||||
}
|
||||
|
||||
void addCopyDockerfileTask(final boolean oss) {
|
||||
task(taskName("copy", oss, "Dockerfile"), type: Copy) {
|
||||
mustRunAfter(taskName("copy", oss, "DockerContext"))
|
||||
into files(oss)
|
||||
|
||||
from('src/docker/Dockerfile') {
|
||||
MavenFilteringHack.filter(it, expansions(oss))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void addBuildDockerImage(final boolean oss) {
|
||||
final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: LoggedExec) {
|
||||
dependsOn taskName("copy", oss, "DockerContext")
|
||||
dependsOn taskName("copy", oss, "Dockerfile")
|
||||
List<String> tags
|
||||
if (oss) {
|
||||
tags = [ "docker.elastic.co/elasticsearch/elasticsearch-oss:${VersionProperties.elasticsearch}" ]
|
||||
} else {
|
||||
tags = [
|
||||
"elasticsearch:${VersionProperties.elasticsearch}",
|
||||
"docker.elastic.co/elasticsearch/elasticsearch:${VersionProperties.elasticsearch}",
|
||||
"docker.elastic.co/elasticsearch/elasticsearch-full:${VersionProperties.elasticsearch}"
|
||||
]
|
||||
}
|
||||
executable 'docker'
|
||||
final List<String> dockerArgs = ['build', files(oss), '--pull']
|
||||
for (final String tag : tags) {
|
||||
dockerArgs.add('--tag')
|
||||
dockerArgs.add(tag)
|
||||
}
|
||||
args dockerArgs.toArray()
|
||||
}
|
||||
BuildPlugin.requireDocker(buildDockerImageTask)
|
||||
}
|
||||
|
||||
for (final boolean oss : [false, true]) {
|
||||
addCopyDockerContextTask(oss)
|
||||
addCopyDockerfileTask(oss)
|
||||
addBuildDockerImage(oss)
|
||||
}
|
||||
|
||||
assemble.dependsOn "buildOssDockerImage"
|
||||
assemble.dependsOn "buildDockerImage"
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
################################################################################
|
||||
# This Dockerfile was generated from the template at distribution/src/docker/Dockerfile
|
||||
#
|
||||
# Beginning of multi stage Dockerfile
|
||||
################################################################################
|
||||
|
||||
################################################################################
|
||||
# Build stage 0 `builder`:
|
||||
# Extract elasticsearch artifact
|
||||
# Install required plugins
|
||||
# Set gid=0 and make group perms==owner perms
|
||||
################################################################################
|
||||
|
||||
FROM centos:7 AS builder
|
||||
|
||||
ENV PATH /usr/share/elasticsearch/bin:$PATH
|
||||
ENV JAVA_HOME /opt/jdk-${jdkVersion}
|
||||
|
||||
RUN curl -s ${jdkUrl} | tar -C /opt -zxf -
|
||||
|
||||
# Replace OpenJDK's built-in CA certificate keystore with the one from the OS
|
||||
# vendor. The latter is superior in several ways.
|
||||
# REF: https://github.com/elastic/elasticsearch-docker/issues/171
|
||||
RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /opt/jdk-${jdkVersion}/lib/security/cacerts
|
||||
|
||||
RUN yum install -y unzip which
|
||||
|
||||
RUN groupadd -g 1000 elasticsearch && \
|
||||
adduser -u 1000 -g 1000 -d /usr/share/elasticsearch elasticsearch
|
||||
|
||||
WORKDIR /usr/share/elasticsearch
|
||||
|
||||
COPY ${elasticsearch} ${ingest-geoip} ${ingest-user-agent} /opt/
|
||||
RUN tar zxf /opt/${elasticsearch} --strip-components=1
|
||||
RUN elasticsearch-plugin install --batch file:///opt/${ingest-geoip}
|
||||
RUN elasticsearch-plugin install --batch file:///opt/${ingest-user-agent}
|
||||
RUN mkdir -p config data logs
|
||||
RUN chmod 0775 config data logs
|
||||
COPY config/elasticsearch.yml config/log4j2.properties config/
|
||||
|
||||
|
||||
################################################################################
|
||||
# Build stage 1 (the actual elasticsearch image):
|
||||
# Copy elasticsearch from stage 0
|
||||
# Add entrypoint
|
||||
################################################################################
|
||||
|
||||
FROM centos:7
|
||||
|
||||
ENV ELASTIC_CONTAINER true
|
||||
ENV JAVA_HOME /opt/jdk-${jdkVersion}
|
||||
|
||||
COPY --from=builder /opt/jdk-${jdkVersion} /opt/jdk-${jdkVersion}
|
||||
|
||||
RUN yum update -y && \
|
||||
yum install -y nc unzip wget which && \
|
||||
yum clean all
|
||||
|
||||
RUN groupadd -g 1000 elasticsearch && \
|
||||
adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \
|
||||
chmod 0775 /usr/share/elasticsearch && \
|
||||
chgrp 0 /usr/share/elasticsearch
|
||||
|
||||
WORKDIR /usr/share/elasticsearch
|
||||
COPY --from=builder --chown=1000:0 /usr/share/elasticsearch /usr/share/elasticsearch
|
||||
ENV PATH /usr/share/elasticsearch/bin:$PATH
|
||||
|
||||
COPY --chown=1000:0 bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
# Openshift overrides USER and uses ones with randomly uid>1024 and gid=0
|
||||
# Allow ENTRYPOINT (and ES) to run even with a different user
|
||||
RUN chgrp 0 /usr/local/bin/docker-entrypoint.sh && \
|
||||
chmod g=u /etc/passwd && \
|
||||
chmod 0775 /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9200 9300
|
||||
|
||||
LABEL org.label-schema.schema-version="1.0" \
|
||||
org.label-schema.vendor="Elastic" \
|
||||
org.label-schema.name="elasticsearch" \
|
||||
org.label-schema.version="${version}" \
|
||||
org.label-schema.url="https://www.elastic.co/products/elasticsearch" \
|
||||
org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \
|
||||
license="${license}"
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
||||
# Dummy overridable parameter parsed by entrypoint
|
||||
CMD ["eswrapper"]
|
||||
|
||||
################################################################################
|
||||
# End of multi-stage Dockerfile
|
||||
################################################################################
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Files created by Elasticsearch should always be group writable too
|
||||
umask 0002
|
||||
|
||||
run_as_other_user_if_needed() {
|
||||
if [[ "$(id -u)" == "0" ]]; then
|
||||
# If running as root, drop to specified UID and run command
|
||||
exec chroot --userspec=1000 / "${@}"
|
||||
else
|
||||
# Either we are running in Openshift with random uid and are a member of the root group
|
||||
# or with a custom --user
|
||||
exec "${@}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Allow user specify custom CMD, maybe bin/elasticsearch itself
|
||||
# for example to directly specify `-E` style parameters for elasticsearch on k8s
|
||||
# or simply to run /bin/bash to check the image
|
||||
if [[ "$1" != "eswrapper" ]]; then
|
||||
if [[ "$(id -u)" == "0" && $(basename "$1") == "elasticsearch" ]]; then
|
||||
# centos:7 chroot doesn't have the `--skip-chdir` option and
|
||||
# changes our CWD.
|
||||
# Rewrite CMD args to replace $1 with `elasticsearch` explicitly,
|
||||
# so that we are backwards compatible with the docs
|
||||
# from the previous Elasticsearch versions<6
|
||||
# and configuration option D:
|
||||
# https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docker.html#_d_override_the_image_8217_s_default_ulink_url_https_docs_docker_com_engine_reference_run_cmd_default_command_or_options_cmd_ulink
|
||||
# Without this, user could specify `elasticsearch -E x.y=z` but
|
||||
# `bin/elasticsearch -E x.y=z` would not work.
|
||||
set -- "elasticsearch" "${@:2}"
|
||||
# Use chroot to switch to UID 1000
|
||||
exec chroot --userspec=1000 / "$@"
|
||||
else
|
||||
# User probably wants to run something else, like /bin/bash, with another uid forced (Openshift?)
|
||||
exec "$@"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Parse Docker env vars to customize Elasticsearch
|
||||
#
|
||||
# e.g. Setting the env var cluster.name=testcluster
|
||||
#
|
||||
# will cause Elasticsearch to be invoked with -Ecluster.name=testcluster
|
||||
#
|
||||
# see https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html#_setting_default_settings
|
||||
|
||||
declare -a es_opts
|
||||
|
||||
while IFS='=' read -r envvar_key envvar_value
|
||||
do
|
||||
# Elasticsearch settings need to have at least two dot separated lowercase
|
||||
# words, e.g. `cluster.name`, except for `processors` which we handle
|
||||
# specially
|
||||
if [[ "$envvar_key" =~ ^[a-z0-9_]+\.[a-z0-9_]+ || "$envvar_key" == "processors" ]]; then
|
||||
if [[ ! -z $envvar_value ]]; then
|
||||
es_opt="-E${envvar_key}=${envvar_value}"
|
||||
es_opts+=("${es_opt}")
|
||||
fi
|
||||
fi
|
||||
done < <(env)
|
||||
|
||||
# The virtual file /proc/self/cgroup should list the current cgroup
|
||||
# membership. For each hierarchy, you can follow the cgroup path from
|
||||
# this file to the cgroup filesystem (usually /sys/fs/cgroup/) and
|
||||
# introspect the statistics for the cgroup for the given
|
||||
# hierarchy. Alas, Docker breaks this by mounting the container
|
||||
# statistics at the root while leaving the cgroup paths as the actual
|
||||
# paths. Therefore, Elasticsearch provides a mechanism to override
|
||||
# reading the cgroup path from /proc/self/cgroup and instead uses the
|
||||
# cgroup path defined the JVM system property
|
||||
# es.cgroups.hierarchy.override. Therefore, we set this value here so
|
||||
# that cgroup statistics are available for the container this process
|
||||
# will run in.
|
||||
export ES_JAVA_OPTS="-Des.cgroups.hierarchy.override=/ $ES_JAVA_OPTS"
|
||||
|
||||
if [[ -d bin/x-pack ]]; then
|
||||
# Check for the ELASTIC_PASSWORD environment variable to set the
|
||||
# bootstrap password for Security.
|
||||
#
|
||||
# This is only required for the first node in a cluster with Security
|
||||
# enabled, but we have no way of knowing which node we are yet. We'll just
|
||||
# honor the variable if it's present.
|
||||
if [[ -n "$ELASTIC_PASSWORD" ]]; then
|
||||
[[ -f /usr/share/elasticsearch/config/elasticsearch.keystore ]] || (run_as_other_user_if_needed elasticsearch-keystore create)
|
||||
if ! (run_as_other_user_if_needed elasticsearch-keystore list | grep -q '^bootstrap.password$'); then
|
||||
(run_as_other_user_if_needed echo "$ELASTIC_PASSWORD" | elasticsearch-keystore add -x 'bootstrap.password')
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$(id -u)" == "0" ]]; then
|
||||
# If requested and running as root, mutate the ownership of bind-mounts
|
||||
if [[ -n "$TAKE_FILE_OWNERSHIP" ]]; then
|
||||
chown -R 1000:0 /usr/share/elasticsearch/{data,logs}
|
||||
fi
|
||||
fi
|
||||
|
||||
run_as_other_user_if_needed /usr/share/elasticsearch/bin/elasticsearch "${es_opts[@]}"
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
cluster.name: "docker-cluster"
|
||||
network.host: 0.0.0.0
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
status = error
|
||||
|
||||
appender.console.type = Console
|
||||
appender.console.name = console
|
||||
appender.console.layout.type = PatternLayout
|
||||
appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
|
||||
|
||||
rootLogger.level = info
|
||||
rootLogger.appenderRef.console.ref = console
|
||||
|
|
@ -20,6 +20,7 @@ List projects = [
|
|||
'distribution:archives:zip',
|
||||
'distribution:archives:oss-tar',
|
||||
'distribution:archives:tar',
|
||||
'distribution:docker',
|
||||
'distribution:packages:oss-deb',
|
||||
'distribution:packages:deb',
|
||||
'distribution:packages:oss-rpm',
|
||||
|
|
|
|||
Loading…
Reference in New Issue