honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support parallel building of Docker images (#52920)

This commit is contained in:
Mark Vieira 2020-02-27 19:18:53 -08:00
parent 82553524af
commit c642a97255
No known key found for this signature in database
GPG Key ID: CA947EF7E6D4B105
4 changed files with 160 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerBuildTask.java Normal file
View File

@ -0,0 +1,133 @@
package org.elasticsearch.gradle.docker;
import org.elasticsearch.gradle.LoggedExec;
import org.gradle.api.DefaultTask;
import org.gradle.api.file.DirectoryProperty;
import org.gradle.api.file.RegularFileProperty;
import org.gradle.api.provider.ListProperty;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputDirectory;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.PathSensitive;
import org.gradle.api.tasks.PathSensitivity;
import org.gradle.api.tasks.TaskAction;
import org.gradle.process.ExecOperations;
import org.gradle.workers.WorkAction;
import org.gradle.workers.WorkParameters;
import org.gradle.workers.WorkerExecutor;
import javax.inject.Inject;
import java.io.IOException;
import java.util.Arrays;
public class DockerBuildTask extends DefaultTask {
private final WorkerExecutor workerExecutor;
private final RegularFileProperty markerFile = getProject().getObjects().fileProperty();
private final DirectoryProperty dockerContext = getProject().getObjects().directoryProperty();
private String[] tags;
private boolean pull = true;
private boolean noCache = true;
@Inject
public DockerBuildTask(WorkerExecutor workerExecutor) {
this.workerExecutor = workerExecutor;
this.markerFile.set(getProject().getLayout().getBuildDirectory().file("markers/" + this.getName() + ".marker"));
}
@TaskAction
public void build() {
workerExecutor.noIsolation().submit(DockerBuildAction.class, params -> {
params.getDockerContext().set(dockerContext);
params.getMarkerFile().set(markerFile);
params.getTags().set(Arrays.asList(tags));
params.getPull().set(pull);
params.getNoCache().set(noCache);
});
}
@InputDirectory
@PathSensitive(PathSensitivity.RELATIVE)
public DirectoryProperty getDockerContext() {
return dockerContext;
}
@Input
public String[] getTags() {
return tags;
}
public void setTags(String[] tags) {
this.tags = tags;
}
@Input
public boolean isPull() {
return pull;
}
public void setPull(boolean pull) {
this.pull = pull;
}
@Input
public boolean isNoCache() {
return noCache;
}
public void setNoCache(boolean noCache) {
this.noCache = noCache;
}
@OutputFile
public RegularFileProperty getMarkerFile() {
return markerFile;
}
public abstract static class DockerBuildAction implements WorkAction<Parameters> {
private final ExecOperations execOperations;
@Inject
public DockerBuildAction(ExecOperations execOperations) {
this.execOperations = execOperations;
}
@Override
public void execute() {
LoggedExec.exec(execOperations, spec -> {
spec.executable("docker");
spec.args("build", getParameters().getDockerContext().get().getAsFile().getAbsolutePath());
if (getParameters().getPull().get()) {
spec.args("--pull");
}
if (getParameters().getNoCache().get()) {
spec.args("--no-cache");
}
getParameters().getTags().get().forEach(tag -> spec.args("--tag", tag));
});
try {
getParameters().getMarkerFile().getAsFile().get().createNewFile();
} catch (IOException e) {
throw new RuntimeException("Failed to create marker file", e);
}
}
}
interface Parameters extends WorkParameters {
DirectoryProperty getDockerContext();
RegularFileProperty getMarkerFile();
ListProperty<String> getTags();
Property<Boolean> getPull();
Property<Boolean> getNoCache();
}
}

23
buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerSupportPlugin.java
View File

@ -3,7 +3,6 @@ package org.elasticsearch.gradle.docker;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.plugins.ExtraPropertiesExtension;
import org.gradle.api.provider.Provider;
import java.io.File;
@ -13,21 +12,10 @@ import java.util.stream.Collectors;
/**
* Plugin providing {@link DockerSupportService} for detecting Docker installations and determining requirements for Docker-based
* Elasticsearch build tasks.
* <p>
* Additionally registers a task graph listener used to assert a compatible Docker installation exists when task requiring Docker are
* scheduled for execution. Tasks may declare a Docker requirement via an extra property. If a compatible Docker installation is not
* available on the build system an exception will be thrown prior to task execution.
*
* <pre>
* task myDockerTask {
* ext.requiresDocker = true
* }
* </pre>
*/
public class DockerSupportPlugin implements Plugin<Project> {
public static final String DOCKER_SUPPORT_SERVICE_NAME = "dockerSupportService";
public static final String DOCKER_ON_LINUX_EXCLUSIONS_FILE = ".ci/dockerOnLinuxExclusions";
public static final String REQUIRES_DOCKER_ATTRIBUTE = "requiresDocker";
@Override
public void apply(Project project) {
@ -45,12 +33,13 @@ public class DockerSupportPlugin implements Plugin<Project> {
)
);
// Ensure that if any tasks declare they require docker, we assert an available Docker installation exists
// Ensure that if we are trying to run any DockerBuildTask tasks, we assert an available Docker installation exists
project.getGradle().getTaskGraph().whenReady(graph -> {
List<String> dockerTasks = graph.getAllTasks().stream().filter(task -> {
ExtraPropertiesExtension ext = task.getExtensions().getExtraProperties();
return ext.has(REQUIRES_DOCKER_ATTRIBUTE) && (boolean) ext.get(REQUIRES_DOCKER_ATTRIBUTE);
}).map(Task::getPath).collect(Collectors.toList());
List<String> dockerTasks = graph.getAllTasks()
.stream()
.filter(task -> task instanceof DockerBuildTask)
.map(Task::getPath)
.collect(Collectors.toList());
if (dockerTasks.isEmpty() == false) {
dockerSupportServiceProvider.get().failIfDockerUnavailable(dockerTasks);

24
buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java
View File

@ -5,9 +5,11 @@ import org.gradle.api.GradleException;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.gradle.api.tasks.Exec;
import org.gradle.api.tasks.Internal;
import org.gradle.process.BaseExecSpec;
import org.gradle.process.ExecOperations;
import org.gradle.process.ExecResult;
import org.gradle.process.ExecSpec;
import org.gradle.process.JavaExecSpec;
@ -30,6 +32,7 @@ import java.util.regex.Pattern;
@SuppressWarnings("unchecked")
public class LoggedExec extends Exec {
private static final Logger LOGGER = Logging.getLogger(LoggedExec.class);
private Consumer<Logger> outputLogger;
public LoggedExec() {
@ -94,21 +97,21 @@ public class LoggedExec extends Exec {
}
public static ExecResult exec(Project project, Action<ExecSpec> action) {
return genericExec(project, project::exec, action);
return genericExec(project::exec, action);
}
public static ExecResult exec(ExecOperations execOperations, Action<ExecSpec> action) {
return genericExec(execOperations::exec, action);
}
public static ExecResult javaexec(Project project, Action<JavaExecSpec> action) {
return genericExec(project, project::javaexec, action);
return genericExec(project::javaexec, action);
}
private static final Pattern NEWLINE = Pattern.compile(System.lineSeparator());
private static <T extends BaseExecSpec> ExecResult genericExec(
Project project,
Function<Action<T>, ExecResult> function,
Action<T> action
) {
if (project.getLogger().isInfoEnabled()) {
private static <T extends BaseExecSpec> ExecResult genericExec(Function<Action<T>, ExecResult> function, Action<T> action) {
if (LOGGER.isInfoEnabled()) {
return function.apply(action);
}
ByteArrayOutputStream output = new ByteArrayOutputStream();
@ -125,7 +128,10 @@ public class LoggedExec extends Exec {
});
} catch (Exception e) {
try {
NEWLINE.splitAsStream(output.toString("UTF-8")).forEach(s -> project.getLogger().error("| " + s));
if (output.size() != 0) {
LOGGER.error("Exec output and error:");
NEWLINE.splitAsStream(output.toString("UTF-8")).forEach(s -> LOGGER.error("| " + s));
}
} catch (UnsupportedEncodingException ue) {
throw new GradleException("Failed to read exec output", ue);
}

22
distribution/docker/build.gradle
View File

@ -1,6 +1,7 @@
import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.docker.DockerBuildTask
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
@ -149,10 +150,11 @@ task integTest(type: Test) {
check.dependsOn integTest
void addBuildDockerImage(final boolean oss) {
final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: LoggedExec) {
ext.requiresDocker = true // mark this task as requiring docker to execute
inputs.files(tasks.named(taskName("copy", oss, "DockerContext")))
List<String> tags
final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: DockerBuildTask) {
TaskProvider<Sync> copyContextTask = tasks.named(taskName("copy", oss, "DockerContext"))
dependsOn(copyContextTask)
dockerContext.fileProvider(copyContextTask.map { it.destinationDir })
if (oss) {
tags = [
"docker.elastic.co/elasticsearch/elasticsearch-oss:${VersionProperties.elasticsearch}",
@ -166,18 +168,6 @@ void addBuildDockerImage(final boolean oss) {
"elasticsearch:test",
]
}
executable 'docker'
final List<String> dockerArgs = ['build', buildPath(oss), '--pull', '--no-cache']
for (final String tag : tags) {
dockerArgs.add('--tag')
dockerArgs.add(tag)
}
args dockerArgs.toArray()
File markerFile = file("build/markers/${it.name}.marker")
outputs.file(markerFile)
doLast {
markerFile.setText('', 'UTF-8')
}
}
assemble.dependsOn(buildDockerImageTask)
}