honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Introduce aarch64 packaging (#53914) (#53926) 

This commit introduces aarch64 packaging, including bundling an aarch64
JDK distribution. We had to make some interesting choices here:
 - ML binaries are not compiled for aarch64, so for now we disable ML on
   aarch64
 - depending on underlying page sizes, we have to disable class data
   sharing
This commit is contained in:
Jason Tedor 2020-03-22 11:58:11 -04:00 committed by GitHub
parent 076ba02e9c
commit 27c8bcbbd1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 258 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/build.gradle
View File

@ -190,8 +190,10 @@ if (project != rootProject) {
distribution project(':distribution:archives:oss-windows-zip')
distribution project(':distribution:archives:darwin-tar')
distribution project(':distribution:archives:oss-darwin-tar')
distribution project(':distribution:archives:linux-aarch64-tar')
distribution project(':distribution:archives:linux-tar')
distribution project(':distribution:archives:oss-linux-tar')
distribution project(':distribution:archives:oss-linux-aarch64-tar')
}
// for external projects we want to remove the marker file indicating we are running the Elasticsearch project

40
buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle;
public enum Architecture {
X64,
AARCH64;
public static Architecture current() {
final String architecture = System.getProperty("os.arch", "");
switch (architecture) {
case "amd64":
case "x86_64":
return X64;
case "aarch64":
return AARCH64;
default:
throw new IllegalArgumentException("can not determine architecture from [" + architecture + "]");
}
}
}

20
buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java
View File

@ -35,6 +35,7 @@ import java.util.regex.Pattern;
public class Jdk implements Buildable, Iterable<File> {
private static final List<String> ALLOWED_ARCHITECTURES = Collections.unmodifiableList(Arrays.asList("aarch64", "x64"));
private static final List<String> ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptopenjdk", "openjdk"));
private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("darwin", "linux", "windows", "mac"));
private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
@ -46,6 +47,7 @@ public class Jdk implements Buildable, Iterable<File> {
private final Property<String> vendor;
private final Property<String> version;
private final Property<String> platform;
private final Property<String> architecture;
private String baseVersion;
private String major;
private String build;
@ -57,6 +59,7 @@ public class Jdk implements Buildable, Iterable<File> {
this.vendor = objectFactory.property(String.class);
this.version = objectFactory.property(String.class);
this.platform = objectFactory.property(String.class);
this.architecture = objectFactory.property(String.class);
}
public String getName() {
@ -99,6 +102,19 @@ public class Jdk implements Buildable, Iterable<File> {
this.platform.set(platform);
}
public String getArchitecture() {
return architecture.get();
}
public void setArchitecture(final String architecture) {
if (ALLOWED_ARCHITECTURES.contains(architecture) == false) {
throw new IllegalArgumentException(
"unknown architecture [" + architecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES
);
}
this.architecture.set(architecture);
}
public String getBaseVersion() {
return baseVersion;
}
@ -167,9 +183,13 @@ public class Jdk implements Buildable, Iterable<File> {
if (vendor.isPresent() == false) {
throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]");
}
if (architecture.isPresent() == false) {
throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]");
}
version.finalizeValue();
platform.finalizeValue();
vendor.finalizeValue();
architecture.finalizeValue();
}
@Override

35
buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
View File

@ -47,7 +47,6 @@ import java.util.Map;
import java.util.concurrent.Callable;
import java.util.stream.StreamSupport;
import static org.elasticsearch.gradle.Util.capitalize;
import static org.elasticsearch.gradle.tool.Boilerplate.findByName;
import static org.elasticsearch.gradle.tool.Boilerplate.maybeCreate;
@ -72,7 +71,10 @@ public class JdkDownloadPlugin implements Plugin<Project> {
DependencyHandler dependencies = project.getDependencies();
Map<String, Object> depConfig = new HashMap<>();
depConfig.put("path", ":"); // root project
depConfig.put("configuration", configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform()));
depConfig.put(
"configuration",
configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture())
);
project.getDependencies().add(jdk.getConfigurationName(), dependencies.project(depConfig));
// ensure a root level jdk download task exists
@ -87,7 +89,14 @@ public class JdkDownloadPlugin implements Plugin<Project> {
}
private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
String extractTaskName = "extract" + capitalize(jdk.getPlatform()) + "Jdk-" + jdk.getVendor() + "-" + jdk.getVersion();
String extractTaskName = String.format(
Locale.ROOT,
"extract-%s-%s-jdk-%s-%s",
jdk.getPlatform(),
jdk.getArchitecture(),
jdk.getVendor(),
jdk.getVersion()
);
// Skip setup if we've already configured a JDK for this platform, vendor and version
if (findByName(rootProject.getTasks(), extractTaskName) == null) {
@ -107,7 +116,7 @@ public class JdkDownloadPlugin implements Plugin<Project> {
repoUrl = "https://artifactory.elstc.co/artifactory/oss-jdk-local/";
artifactPattern = String.format(
Locale.ROOT,
"adoptopenjdk/OpenJDK%sU-jdk_x64_[module]_hotspot_[revision]_%s.[ext]",
"adoptopenjdk/OpenJDK%sU-jdk_[classifier]_[module]_hotspot_[revision]_%s.[ext]",
jdk.getMajor(),
jdk.getBuild()
);
@ -121,14 +130,14 @@ public class JdkDownloadPlugin implements Plugin<Project> {
+ jdk.getHash()
+ "/"
+ jdk.getBuild()
+ "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
+ "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
} else {
// simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
artifactPattern = "java/GA/jdk"
+ jdk.getMajor()
+ "/"
+ jdk.getBuild()
+ "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
+ "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
}
} else {
throw new GradleException("Unknown JDK vendor [" + jdk.getVendor() + "]");
@ -150,14 +159,14 @@ public class JdkDownloadPlugin implements Plugin<Project> {
// Declare a configuration and dependency from which to download the remote JDK
final ConfigurationContainer configurations = rootProject.getConfigurations();
String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform(), jdk.getArchitecture());
Configuration downloadConfiguration = maybeCreate(configurations, downloadConfigName);
rootProject.getDependencies().add(downloadConfigName, dependencyNotation(jdk));
// Create JDK extract task
final Provider<Directory> extractPath = rootProject.getLayout()
.getBuildDirectory()
.dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform());
.dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform() + "_" + jdk.getArchitecture());
TaskProvider<?> extractTask = createExtractTask(
extractTaskName,
@ -168,7 +177,13 @@ public class JdkDownloadPlugin implements Plugin<Project> {
);
// Declare a configuration for the extracted JDK archive
String artifactConfigName = configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
String artifactConfigName = configName(
"extracted_jdk",
jdk.getVendor(),
jdk.getVersion(),
jdk.getPlatform(),
jdk.getArchitecture()
);
maybeCreate(configurations, artifactConfigName);
rootProject.getArtifacts().add(artifactConfigName, extractPath, artifact -> artifact.builtBy(extractTask));
}
@ -254,7 +269,7 @@ public class JdkDownloadPlugin implements Plugin<Project> {
: jdk.getPlatform();
String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + ":" + jdk.getArchitecture() + "@" + extension;
}
private static String groupName(Jdk jdk) {

6
buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java
View File

@ -173,12 +173,14 @@ public class DistroTestPlugin implements Plugin<Project> {
String name,
String vendor,
String version,
String platform
String platform,
String architecture
) {
Jdk jdk = jdksContainer.create(name);
jdk.setVendor(vendor);
jdk.setVersion(version);
jdk.setPlatform(platform);
jdk.setArchitecture(architecture);
return jdk;
}
@ -210,7 +212,7 @@ public class DistroTestPlugin implements Plugin<Project> {
NamedDomainObjectContainer<Jdk> jdksContainer = JdkDownloadPlugin.getContainer(project);
String platform = box.contains("windows") ? "windows" : "linux";
Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform);
Jdk gradleJdk = createJdk(jdksContainer, "gradle", GRADLE_JDK_VENDOR, GRADLE_JDK_VERSION, platform, "x64");
// setup VM used by these tests
VagrantExtension vagrant = project.getExtensions().getByType(VagrantExtension.class);

2
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.gradle.testclusters;
import org.elasticsearch.gradle.Architecture;
import org.elasticsearch.gradle.DistributionDownloadPlugin;
import org.elasticsearch.gradle.Jdk;
import org.elasticsearch.gradle.JdkDownloadPlugin;
@ -65,6 +66,7 @@ public class TestClustersPlugin implements Plugin<Project> {
jdk.setVendor(LEGACY_JAVA_VENDOR);
jdk.setVersion(LEGACY_JAVA_VERSION);
jdk.setPlatform(OS.current().name().toLowerCase());
jdk.setArchitecture(Architecture.current().name().toLowerCase());
});
// enable the DSL to describe clusters

51
buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java
View File

@ -36,7 +36,7 @@ public class JdkDownloadPluginTests extends GradleUnitTestCase {
}
public void testMissingVendor() {
assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "vendor not specified for jdk [testjdk]");
assertJdkError(createProject(), "testjdk", null, "11.0.2+33", "linux", "x64", "vendor not specified for jdk [testjdk]");
}
public void testUnknownVendor() {
@ -46,20 +46,29 @@ public class JdkDownloadPluginTests extends GradleUnitTestCase {
"unknown",
"11.0.2+33",
"linux",
"x64",
"unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk]"
);
}
public void testMissingVersion() {
assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "version not specified for jdk [testjdk]");
assertJdkError(createProject(), "testjdk", "openjdk", null, "linux", "x64", "version not specified for jdk [testjdk]");
}
public void testBadVersionFormat() {
assertJdkError(createProject(), "testjdk", "openjdk", "badversion", "linux", "malformed version [badversion] for jdk [testjdk]");
assertJdkError(
createProject(),
"testjdk",
"openjdk",
"badversion",
"linux",
"x64",
"malformed version [badversion] for jdk [testjdk]"
);
}
public void testMissingPlatform() {
assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "platform not specified for jdk [testjdk]");
assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", null, "x64", "platform not specified for jdk [testjdk]");
}
public void testUnknownPlatform() {
@ -69,19 +78,44 @@ public class JdkDownloadPluginTests extends GradleUnitTestCase {
"openjdk",
"11.0.2+33",
"unknown",
"x64",
"unknown platform [unknown] for jdk [testjdk], must be one of [darwin, linux, windows, mac]"
);
}
private void assertJdkError(Project project, String name, String vendor, String version, String platform, String message) {
public void testMissingArchitecture() {
assertJdkError(createProject(), "testjdk", "openjdk", "11.0.2+33", "linux", null, "architecture not specified for jdk [testjdk]");
}
public void testUnknownArchitecture() {
assertJdkError(
createProject(),
"testjdk",
"openjdk",
"11.0.2+33",
"linux",
"unknown",
"unknown architecture [unknown] for jdk [testjdk], must be one of [aarch64, x64]"
);
}
private void assertJdkError(
final Project project,
final String name,
final String vendor,
final String version,
final String platform,
final String architecture,
final String message
) {
IllegalArgumentException e = expectThrows(
IllegalArgumentException.class,
() -> createJdk(project, name, vendor, version, platform)
() -> createJdk(project, name, vendor, version, platform, architecture)
);
assertThat(e.getMessage(), equalTo(message));
}
private void createJdk(Project project, String name, String vendor, String version, String platform) {
private void createJdk(Project project, String name, String vendor, String version, String platform, String architecture) {
@SuppressWarnings("unchecked")
NamedDomainObjectContainer<Jdk> jdks = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
jdks.create(name, jdk -> {
@ -94,6 +128,9 @@ public class JdkDownloadPluginTests extends GradleUnitTestCase {
if (platform != null) {
jdk.setPlatform(platform);
}
if (architecture != null) {
jdk.setArchitecture(architecture);
}
}).finalizeValues();
}

1
buildSrc/src/testKit/jdk-download/reuse/build.gradle
View File

@ -7,5 +7,6 @@ jdks {
vendor = fakeJdkVendor
version = fakeJdkVersion
platform = "linux"
architeecture = "x64"
}
}

3
buildSrc/src/testKit/jdk-download/subproj/build.gradle
View File

@ -10,16 +10,19 @@ jdks {
vendor = fakeJdkVendor
version = fakeJdkVersion
platform = "linux"
architecture = "x64"
}
darwin {
vendor = fakeJdkVendor
version = fakeJdkVersion
platform = "darwin"
architecture = "x64"
}
windows {
vendor = fakeJdkVendor
version = fakeJdkVersion
platform = "windows"
architecture = "x64"
}
}

42
distribution/archives/build.gradle
View File

@ -52,7 +52,7 @@ task createJvmOptionsDir(type: EmptyDirTask) {
dirMode = 0750
}
CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) {
CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, boolean oss, boolean jdk) {
return copySpec {
into("elasticsearch-${version}") {
into('lib') {
@ -72,7 +72,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
}
if (jdk) {
into("darwin".equals(platform) ? 'jdk.app' : 'jdk') {
with jdkFiles(project, platform)
with jdkFiles(project, platform, architecture)
}
}
into('') {
@ -118,31 +118,31 @@ Closure commonZipConfig = {
task buildIntegTestZip(type: Zip) {
configure(commonZipConfig)
with archiveFiles(transportModulesFiles, 'zip', null, true, false)
with archiveFiles(transportModulesFiles, 'zip', null, 'x64', true, false)
}
task buildWindowsZip(type: Zip) {
configure(commonZipConfig)
archiveClassifier = 'windows-x86_64'
with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', false, true)
with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', 'x64', false, true)
}
task buildOssWindowsZip(type: Zip) {
configure(commonZipConfig)
archiveClassifier = 'windows-x86_64'
with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', true, true)
with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', 'x64', true, true)
}
task buildNoJdkWindowsZip(type: Zip) {
configure(commonZipConfig)
archiveClassifier = 'no-jdk-windows-x86_64'
with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', false, false)
with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', 'x64', false, false)
}
task buildOssNoJdkWindowsZip(type: Zip) {
configure(commonZipConfig)
archiveClassifier = 'no-jdk-windows-x86_64'
with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', true, false)
with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', 'x64', true, false)
}
Closure commonTarConfig = {
@ -155,49 +155,61 @@ Closure commonTarConfig = {
task buildDarwinTar(type: SymbolicLinkPreservingTar) {
configure(commonTarConfig)
archiveClassifier = 'darwin-x86_64'
with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', false, true)
with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', 'x64', false, true)
}
task buildOssDarwinTar(type: SymbolicLinkPreservingTar) {
configure(commonTarConfig)
archiveClassifier = 'darwin-x86_64'
with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', true, true)
with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', 'x64', true, true)
}
task buildNoJdkDarwinTar(type: SymbolicLinkPreservingTar) {
configure(commonTarConfig)
archiveClassifier = 'no-jdk-darwin-x86_64'
with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', false, false)
with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', 'x64', false, false)
}
task buildOssNoJdkDarwinTar(type: SymbolicLinkPreservingTar) {
configure(commonTarConfig)
archiveClassifier = 'no-jdk-darwin-x86_64'
with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', true, false)
with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', 'x64', true, false)
}
task buildLinuxAarch64Tar(type: SymbolicLinkPreservingTar) {
configure(commonTarConfig)
archiveClassifier = 'linux-aarch64'
with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'aarch64', false, true)
}
task buildLinuxTar(type: SymbolicLinkPreservingTar) {
configure(commonTarConfig)
archiveClassifier = 'linux-x86_64'
with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', false, true)
with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'x64', false, true)
}
task buildOssLinuxAarch64Tar(type: SymbolicLinkPreservingTar) {
configure(commonTarConfig)
archiveClassifier = 'linux-x86_64'
with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, true)
}
task buildOssLinuxTar(type: SymbolicLinkPreservingTar) {
configure(commonTarConfig)
archiveClassifier = 'linux-x86_64'
with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', true, true)
with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, true)
}
task buildNoJdkLinuxTar(type: SymbolicLinkPreservingTar) {
configure(commonTarConfig)
archiveClassifier = 'no-jdk-linux-x86_64'
with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', false, false)
with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', 'x64', false, false)
}
task buildOssNoJdkLinuxTar(type: SymbolicLinkPreservingTar) {
configure(commonTarConfig)
archiveClassifier = 'no-jdk-linux-x86_64'
with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', true, false)
with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', 'x64', true, false)
}
Closure tarExists = { it -> new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }

2
distribution/archives/linux-aarch64-tar/build.gradle Normal file
View File

@ -0,0 +1,2 @@
// This file is intentionally blank. All configuration of the
// distribution is done in the parent project.

2
distribution/archives/oss-linux-aarch64-tar/build.gradle Normal file
View File

@ -0,0 +1,2 @@
// This file is intentionally blank. All configuration of the
// distribution is done in the parent project.

10
distribution/build.gradle
View File

@ -388,16 +388,17 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
}
}
jdkFiles = { Project project, String platform ->
jdkFiles = { Project project, String platform, String architecture ->
project.jdks {
"bundled_${platform}" {
"bundled_${platform}_${architecture}" {
it.platform = platform
it.version = VersionProperties.getBundledJdk(platform)
it.vendor = VersionProperties.bundledJdkVendor
it.architecture = architecture
}
}
return copySpec {
from project.jdks."bundled_${platform}"
from project.jdks."bundled_${platform}_${architecture}"
exclude "demo/**"
eachFile { FileCopyDetails details ->
if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') {
@ -605,10 +606,13 @@ subprojects {
['archives:windows-zip', 'archives:oss-windows-zip',
'archives:darwin-tar', 'archives:oss-darwin-tar',
'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar',
'archives:linux-tar', 'archives:oss-linux-tar',
'archives:integ-test-zip',
'packages:rpm', 'packages:deb',
'packages:aarch64-rpm', 'packages:aarch64-deb',
'packages:oss-rpm', 'packages:oss-deb',
'packages:aarch64-oss-rpm', 'packages:aarch64-oss-deb'
].forEach { subName ->
Project subproject = project("${project.path}:${subName}")
Configuration configuration = configurations.create(subproject.name)

2
distribution/packages/aarch64-deb/build.gradle Normal file
View File

@ -0,0 +1,2 @@
// This file is intentionally blank. All configuration of the
// distribution is done in the parent project.

2
distribution/packages/aarch64-oss-deb/build.gradle Normal file
View File

@ -0,0 +1,2 @@
// This file is intentionally blank. All configuration of the
// distribution is done in the parent project.

2
distribution/packages/aarch64-oss-rpm/build.gradle Normal file
View File

@ -0,0 +1,2 @@
// This file is intentionally blank. All configuration of the
// distribution is done in the parent project.

2
distribution/packages/aarch64-rpm/build.gradle Normal file
View File

@ -0,0 +1,2 @@
// This file is intentionally blank. All configuration of the
// distribution is done in the parent project.

50
distribution/packages/build.gradle
View File

@ -98,17 +98,17 @@ addProcessFilesTask('rpm', false, false)
// Common configuration that is package dependent. This can't go in ospackage
// since we have different templated files that need to be consumed, but the structure
// is the same
Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
Closure commonPackageConfig(String type, boolean oss, boolean jdk, String architecture) {
return {
onlyIf {
OS.current().equals(OS.WINDOWS) == false
}
dependsOn "process${oss ? 'Oss' : ''}${jdk ? '' : 'NoJdk'}${type.capitalize()}Files"
packageName "elasticsearch${oss ? '-oss' : ''}"
arch(type == 'deb' ? 'amd64' : 'X86_64')
arch(architecture == 'aarch64' ? 'aarch64' : type == 'deb' ? 'amd64' : 'X86_64')
// Follow elasticsearch's file naming convention
String jdkString = jdk ? "" : "no-jdk-"
String prefix = "${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}"
String prefix = "${architecture == 'aarch64' ? 'aarch64-' : ''}${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}"
destinationDir = file("${prefix}/build/distributions")
// SystemPackagingTask overrides default archive task convention mappings, but doesn't provide a setter so we have to override the convention mapping itself
@ -143,7 +143,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
}
if (jdk) {
into('jdk') {
with jdkFiles(project, 'linux')
with jdkFiles(project, 'linux', architecture)
}
}
// we need to specify every intermediate directory in these paths so the package managers know they are explicitly
@ -306,9 +306,9 @@ ospackage {
into '/usr/share/elasticsearch'
}
Closure commonDebConfig(boolean oss, boolean jdk) {
Closure commonDebConfig(boolean oss, boolean jdk, String architecture) {
return {
configure(commonPackageConfig('deb', oss, jdk))
configure(commonPackageConfig('deb', oss, jdk, architecture))
// jdeb does not provide a way to set the License control attribute, and ospackage
// silently ignores setting it. Instead, we set the license as "custom field"
@ -336,25 +336,33 @@ Closure commonDebConfig(boolean oss, boolean jdk) {
}
}
task buildAarch64Deb(type: Deb) {
configure(commonDebConfig(false, true, 'aarch64'))
}
task buildDeb(type: Deb) {
configure(commonDebConfig(false, true))
configure(commonDebConfig(false, true, 'x64'))
}
task buildAarch64OssDeb(type: Deb) {
configure(commonDebConfig(true, true, 'aarch64'))
}
task buildOssDeb(type: Deb) {
configure(commonDebConfig(true, true))
configure(commonDebConfig(true, true, 'x64'))
}
task buildNoJdkDeb(type: Deb) {
configure(commonDebConfig(false, false))
configure(commonDebConfig(false, false, 'x64'))
}
task buildOssNoJdkDeb(type: Deb) {
configure(commonDebConfig(true, false))
configure(commonDebConfig(true, false, 'x64'))
}
Closure commonRpmConfig(boolean oss, boolean jdk) {
Closure commonRpmConfig(boolean oss, boolean jdk, String architecture) {
return {
configure(commonPackageConfig('rpm', oss, jdk))
configure(commonPackageConfig('rpm', oss, jdk, architecture))
if (oss) {
license 'ASL 2.0'
@ -381,20 +389,28 @@ Closure commonRpmConfig(boolean oss, boolean jdk) {
}
}
task buildAarch64Rpm(type: Rpm) {
configure(commonRpmConfig(false, true, 'aarch64'))
}
task buildRpm(type: Rpm) {
configure(commonRpmConfig(false, true))
configure(commonRpmConfig(false, true, 'x64'))
}
task buildAarch64OssRpm(type: Rpm) {
configure(commonRpmConfig(true, true, 'aarch64'))
}
task buildOssRpm(type: Rpm) {
configure(commonRpmConfig(true, true))
configure(commonRpmConfig(true, true, 'x64'))
}
task buildNoJdkRpm(type: Rpm) {
configure(commonRpmConfig(false, false))
configure(commonRpmConfig(false, false, 'x64'))
}
task buildOssNoJdkRpm(type: Rpm) {
configure(commonRpmConfig(true, false))
configure(commonRpmConfig(true, false, 'x64'))
}
Closure dpkgExists = { it -> new File('/bin/dpkg-deb').exists() || new File('/usr/bin/dpkg-deb').exists() || new File('/usr/local/bin/dpkg-deb').exists() }
@ -446,6 +462,8 @@ subprojects {
final File rpmDatabase = new File(extractionDir, 'rpm-database')
commandLine 'rpm',
'--badreloc',
'--ignorearch',
'--ignoreos',
'--nodeps',
'--noscripts',
'--notriggers',

6
distribution/src/bin/elasticsearch
View File

@ -29,7 +29,7 @@ for option in "$@"; do
done
if [ -z "$ES_TMPDIR" ]; then
ES_TMPDIR=`"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory`
ES_TMPDIR=`"$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory`
fi
# get keystore password before setting java options to avoid
@ -52,12 +52,13 @@ fi
# - second, JVM options are read from jvm.options and jvm.options.d/*.options
# - third, JVM options from ES_JAVA_OPTS are applied
# - fourth, ergonomic JVM options are applied
ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"`
ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"`
# manual parsing to find out, if process should be detached
if [[ $DAEMONIZE = false ]]; then
exec \
"$JAVA" \
"$XSHARE" \
$ES_JAVA_OPTS \
-Des.path.home="$ES_HOME" \
-Des.path.conf="$ES_PATH_CONF" \
@ -70,6 +71,7 @@ if [[ $DAEMONIZE = false ]]; then
else
exec \
"$JAVA" \
"$XSHARE" \
$ES_JAVA_OPTS \
-Des.path.home="$ES_HOME" \
-Des.path.conf="$ES_PATH_CONF" \

1
distribution/src/bin/elasticsearch-cli
View File

@ -22,6 +22,7 @@ ES_JAVA_OPTS="-Xms4m -Xmx64m -XX:+UseSerialGC ${ES_JAVA_OPTS}"
exec \
"$JAVA" \
"$XSHARE" \
$ES_JAVA_OPTS \
-Des.path.home="$ES_HOME" \
-Des.path.conf="$ES_PATH_CONF" \

8
distribution/src/bin/elasticsearch-env
View File

@ -67,8 +67,14 @@ if [ ! -z "$JAVA_OPTS" ]; then
echo "pass JVM parameters via ES_JAVA_OPTS"
fi
if [[ "$("$JAVA" -version 2>/dev/null)" =~ "Unable to map CDS archive" ]]; then
XSHARE="-Xshare:off"
else
XSHARE="-Xshare:auto"
fi
# check the Java version
"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.java_version_checker.JavaVersionChecker
"$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.java_version_checker.JavaVersionChecker
export HOSTNAME=$HOSTNAME

11
distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java
View File

@ -103,10 +103,13 @@ final class JvmErgonomics {
*/
final String java = Paths.get(System.getProperty("java.home"), "bin", "java").toString();
final List<String> command = Collections.unmodifiableList(
Stream.of(Stream.of(java), userDefinedJvmOptions.stream(), Stream.of("-XX:+PrintFlagsFinal"), Stream.of("-version"))
.reduce(Stream::concat)
.get()
.collect(Collectors.toList())
Stream.of(
Stream.of(java),
userDefinedJvmOptions.stream(),
Stream.of("-Xshare:off"),
Stream.of("-XX:+PrintFlagsFinal"),
Stream.of("-version")
).reduce(Stream::concat).get().collect(Collectors.toList())
);
final Process process = new ProcessBuilder().command(command).start();
final List<String> output = readLinesFromInputStream(process.getInputStream());

2
gradle/runtime-jdk-provision.gradle
View File

@ -1,3 +1,4 @@
import org.elasticsearch.gradle.Architecture
import org.elasticsearch.gradle.OS
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.info.BuildParams
@ -9,6 +10,7 @@ jdks {
vendor = VersionProperties.bundledJdkVendor
version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase())
platform = OS.current().name().toLowerCase()
architecture = Architecture.current().name().toLowerCase()
}
}

6
settings.gradle
View File

@ -27,7 +27,9 @@ List projects = [
'distribution:archives:darwin-tar',
'distribution:archives:oss-no-jdk-darwin-tar',
'distribution:archives:no-jdk-darwin-tar',
'distribution:archives:oss-linux-aarch64-tar',
'distribution:archives:oss-linux-tar',
'distribution:archives:linux-aarch64-tar',
'distribution:archives:linux-tar',
'distribution:archives:oss-no-jdk-linux-tar',
'distribution:archives:no-jdk-linux-tar',
@ -36,11 +38,15 @@ List projects = [
'distribution:docker:docker-export',
'distribution:docker:oss-docker-build-context',
'distribution:docker:oss-docker-export',
'distribution:packages:aarch64-oss-deb',
'distribution:packages:oss-deb',
'distribution:packages:aarch64-deb',
'distribution:packages:deb',
'distribution:packages:oss-no-jdk-deb',
'distribution:packages:no-jdk-deb',
'distribution:packages:aarch64-oss-rpm',
'distribution:packages:oss-rpm',
'distribution:packages:aarch64-rpm',
'distribution:packages:rpm',
'distribution:packages:oss-no-jdk-rpm',
'distribution:packages:no-jdk-rpm',

9
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
View File

@ -71,7 +71,14 @@ public class XPackSettings {
public static final Setting<Boolean> GRAPH_ENABLED = Setting.boolSetting("xpack.graph.enabled", true, Setting.Property.NodeScope);
/** Setting for enabling or disabling machine learning. Defaults to true. */
public static final Setting<Boolean> MACHINE_LEARNING_ENABLED = Setting.boolSetting("xpack.ml.enabled", true,
public static final Setting<Boolean> MACHINE_LEARNING_ENABLED = Setting.boolSetting(
"xpack.ml.enabled",
"aarch64".equals(System.getProperty("os.arch")) ? false : true,
value -> {
if (value && "aarch64".equals(System.getProperty("os.arch"))) {
throw new IllegalArgumentException("[xpack.ml.enabled] can not be set to [true] on [aarch64]");
}
},
Setting.Property.NodeScope);
/** Setting for enabling or disabling rollup. Defaults to true. */

1
x-pack/plugin/sql/src/main/bin/elasticsearch-sql-cli
View File

@ -12,5 +12,6 @@ CLI_JAR=$(ls "$ES_HOME"/bin/elasticsearch-sql-cli-*.jar)
exec \
"$JAVA" \
"$XSHARE" \
-jar "$CLI_JAR" \
"$@"