honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix slow sync test clustres artifacts task (#42012) 

* Fix slow sync test clustres artifacts task

The task was mistakenly adding a combinational explosion of task
actions  all doing the same thing.
With this PR this is fixed and each version - distribution pair is only
extracted once.
I appologieze for the SSD wear.

* Look for configurations on the root project
* Add dependency on configurations
* This should be a `copy` so we don't blow away all the other distros
* Don't copy example plugin build directory in integration tests
This commit is contained in:
Alpar Torok 2019-05-10 14:24:42 +03:00
parent ea7db2bb6a
commit db8fe1de00
2 changed files with 36 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
View File

@ -182,8 +182,9 @@ public class TestClustersPlugin implements Plugin<Project> {
claimsInventory.put(elasticsearchCluster, claimsInventory.getOrDefault(elasticsearchCluster, 0) + 1);
}
}));
logger.info("Claims inventory: {}", claimsInventory);
if (claimsInventory.isEmpty() == false) {
logger.info("Claims inventory: {}", claimsInventory);
}
});
}
@ -279,8 +280,14 @@ public class TestClustersPlugin implements Plugin<Project> {
// the clusters will look for artifacts there based on the naming conventions.
// Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in
// the build.
Task sync = Boilerplate.maybeCreate(rootProject.getTasks(), SYNC_ARTIFACTS_TASK_NAME, onCreate -> {
Boilerplate.maybeCreate(rootProject.getTasks(), SYNC_ARTIFACTS_TASK_NAME, onCreate -> {
onCreate.getOutputs().dir(getExtractDir(rootProject));
onCreate.getInputs().files(
project.getRootProject().getConfigurations().matching(conf -> conf.getName().startsWith(HELPER_CONFIGURATION_PREFIX))
);
onCreate.dependsOn(project.getRootProject().getConfigurations()
.matching(conf -> conf.getName().startsWith(HELPER_CONFIGURATION_PREFIX))
);
// NOTE: Gradle doesn't allow a lambda here ( fails at runtime )
onCreate.doFirst(new Action<Task>() {
@Override
@ -290,6 +297,31 @@ public class TestClustersPlugin implements Plugin<Project> {
project.delete(getExtractDir(rootProject));
}
});
onCreate.doLast(new Action<Task>() {
@Override
public void execute(Task task) {
project.getRootProject().getConfigurations()
.matching(config -> config.getName().startsWith(HELPER_CONFIGURATION_PREFIX))
.forEach(config -> project.copy(spec ->
config.getResolvedConfiguration()
.getResolvedArtifacts()
.forEach(resolvedArtifact -> {
final FileTree files;
File file = resolvedArtifact.getFile();
if (file.getName().endsWith(".zip")) {
files = project.zipTree(file);
} else if (file.getName().endsWith("tar.gz")) {
files = project.tarTree(file);
} else {
throw new IllegalArgumentException("Can't extract " + file + " unknown file extension");
}
logger.info("Extracting {}@{}", resolvedArtifact, config);
spec.from(files, s -> s.into(resolvedArtifact.getModuleVersion().getId().getGroup()));
spec.into(getExtractDir(project));
}))
);
}
});
});
// When the project evaluated we know of all tasks that use clusters.
@ -347,29 +379,6 @@ public class TestClustersPlugin implements Plugin<Project> {
distribution.getFileExtension());
}
sync.getInputs().files(helperConfiguration);
// NOTE: Gradle doesn't allow a lambda here ( fails at runtime )
sync.doLast(new Action<Task>() {
@Override
public void execute(Task task) {
project.copy(spec ->
helperConfiguration.getResolvedConfiguration().getResolvedArtifacts().forEach(resolvedArtifact -> {
final FileTree files;
File file = resolvedArtifact.getFile();
if (file.getName().endsWith(".zip")) {
files = project.zipTree(file);
} else if (file.getName().endsWith("tar.gz")) {
files = project.tarTree(file);
} else {
throw new IllegalArgumentException("Can't extract " + file + " unknown file extension");
}
spec.from(files, s -> s.into(resolvedArtifact.getModuleVersion().getId().getGroup()));
spec.into(getExtractDir(project));
}));
}
});
})));
}

3
buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java
View File

@ -75,7 +75,7 @@ public class BuildExamplePluginsIT extends GradleIntegrationTestCase {
}
public void testCurrentExamplePlugin() throws IOException {
FileUtils.copyDirectory(examplePlugin, tmpDir.getRoot());
FileUtils.copyDirectory(examplePlugin, tmpDir.getRoot(), pathname -> pathname.getPath().contains("/build/") == false);
adaptBuildScriptForTest();
@ -156,5 +156,4 @@ public class BuildExamplePluginsIT extends GradleIntegrationTestCase {
throw new RuntimeException(e);
}
}
}