honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'es/7.x' into enrich-7.x

This commit is contained in:
Martijn van Groningen 2019-05-09 13:33:34 +02:00
parent 202a840da9 37771502ae
commit 44f09a9a86
No known key found for this signature in database
GPG Key ID: AB236F4FCF2AF12A
19 changed files with 238 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Vagrantfile vendored
View File

@ -161,13 +161,17 @@ def deb_common(config, name, extra: '')
s.privileged = false
s.inline = "sudo sed -i '/tty/!s/mesg n/tty -s \\&\\& mesg n/' /root/.profile"
end
extra_with_lintian = <<-SHELL
install lintian
#{extra}
SHELL
linux_common(
config,
name,
update_command: 'apt-get update',
update_tracking_file: '/var/cache/apt/archives/last_update',
install_command: 'apt-get install -y',
extra: extra
extra: extra_with_lintian
)
end

5
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
View File

@ -34,6 +34,9 @@ import org.gradle.api.plugins.quality.Checkstyle
class PrecommitTasks {
/** Adds a precommit task, which depends on non-test verification tasks. */
public static final String CHECKSTYLE_VERSION = '8.20'
public static Task create(Project project, boolean includeDependencyLicenses) {
project.configurations.create("forbiddenApisCliJar")
project.dependencies {
@ -214,7 +217,7 @@ class PrecommitTasks {
configProperties = [
suppressions: checkstyleSuppressions
]
toolVersion = '8.10.1'
toolVersion = CHECKSTYLE_VERSION
}
project.tasks.withType(Checkstyle) { task ->

2
buildSrc/src/main/resources/minimumGradleVersion
View File

@ -1 +1 @@
5.3
5.4.1

17
distribution/packages/build.gradle
View File

@ -270,13 +270,13 @@ apply plugin: 'nebula.ospackage-base'
// this is package indepdendent configuration
ospackage {
maintainer 'Elasticsearch Team <info@elastic.co>'
summary '''
Elasticsearch is a distributed RESTful search engine built for the cloud.
summary 'Distributed RESTful search engine built for the cloud'
packageDescription '''
Reference documentation can be found at
https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html
and the 'Elasticsearch: The Definitive Guide' book can be found at
https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html
'''.stripIndent().replace('\n', ' ').trim()
'''.stripIndent().trim()
url 'https://www.elastic.co/'
// signing setup
@ -288,7 +288,8 @@ ospackage {
new File(new File(System.getProperty('user.home'), '.gnupg'), 'secring.gpg')
}
requires('coreutils')
// version found on oldest supported distro, centos-6
requires('coreutils', '8.4', GREATER | EQUAL)
fileMode 0644
dirMode 0755
@ -312,12 +313,18 @@ Closure commonDebConfig(boolean oss, boolean jdk) {
version = project.version.replace('-', '~')
packageGroup 'web'
requires 'bash'
// versions found on oldest supported distro, centos-6
requires('bash', '4.1', GREATER | EQUAL)
requires('lsb-base', '4', GREATER | EQUAL)
requires 'libc6'
requires 'adduser'
into('/usr/share/lintian/overrides') {
from('src/deb/lintian/elasticsearch')
if (oss) {
rename('elasticsearch', 'elasticsearch-oss')
}
}
}
}

56
distribution/packages/src/deb/lintian/elasticsearch
View File

@ -1,8 +1,48 @@
# Ignore arch dependent warnings, we chose the right libs on start
elasticsearch binary: arch-independent-package-contains-binary-or-object
# Not stripping external libraries
elasticsearch binary: unstripped-binary-or-object
# Ignore arch dependent warnings, we chose the right libs on start
elasticsearch binary: arch-dependent-file-in-usr-share
# Please check our changelog at http://www.elastic.co/downloads/elasticsearch
elasticsearch binary: changelog-file-missing-in-native-package
# we don't have a changelog, but we put our copyright file
# under /usr/share/doc/elasticsearch, which triggers this warning
changelog-file-missing-in-native-package
# we intentionally copy our copyright file for all deb packages
copyright-file-contains-full-apache-2-license
copyright-should-refer-to-common-license-file-for-apache-2
copyright-without-copyright-notice
# we still put all our files under /usr/share/elasticsearch even after transition to platform dependent packages
arch-dependent-file-in-usr-share
# we have a bundled jdk, so don't use jarwrapper
missing-dep-on-jarwrapper
# we prefer to not make our config and log files world readable
non-standard-file-perm etc/default/elasticsearch 0660 != 0644
non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
non-standard-file-perm etc/elasticsearch/*
non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755
executable-is-not-world-readable etc/init.d/elasticsearch 0750
non-standard-file-permissions-for-etc-init.d-script etc/init.d/elasticsearch 0750 != 0755
# this lintian tag is simply wrong; contrary to the explanation, debian systemd
# does actually look at /usr/lib/systemd/system
systemd-service-file-outside-lib usr/lib/systemd/system/elasticsearch.service
# we do not automatically enable the service in init.d or systemd
script-in-etc-init.d-not-registered-via-update-rc.d etc/init.d/elasticsearch
# the package scripts handle init.d/systemd directly and don't need to use deb helpers
maintainer-script-calls-systemctl
prerm-calls-updaterc.d elasticsearch
# bundled JDK
embedded-library
arch-dependent-file-in-usr-share usr/share/elasticsearch/jdk/*
unstripped-binary-or-object usr/share/elasticsearch/jdk/*
extra-license-file usr/share/elasticsearch/jdk/legal/*
hardening-no-pie usr/share/elasticsearch/jdk/bin/*
hardening-no-pie usr/share/elasticsearch/jdk/lib/*
# the system java version that lintian assumes is far behind what elasticsearch uses
unknown-java-class-version
# elastic licensed modules contain elastic license
extra-license-file usr/share/elasticsearch/modules/*

2
distribution/src/bin/elasticsearch
View File

@ -18,7 +18,7 @@ source "`dirname "$0"`"/elasticsearch-env
ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options
JVM_OPTIONS=`"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_JVM_OPTIONS"`
ES_JAVA_OPTS="${JVM_OPTIONS//\$\{ES_TMPDIR\}/$ES_TMPDIR} $ES_JAVA_OPTS"
ES_JAVA_OPTS="${JVM_OPTIONS//\$\{ES_TMPDIR\}/$ES_TMPDIR}"
# manual parsing to find out, if process should be detached
if ! echo $* | grep -E '(^-d |-d$| -d |--daemonize$|--daemonize )' > /dev/null; then

2
distribution/src/bin/elasticsearch-service.bat
View File

@ -112,7 +112,7 @@ if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
@setlocal
for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" || echo jvm_options_parser_failed"`) do set JVM_OPTIONS=%%a
@endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%JVM_OPTIONS%" & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!% %ES_JAVA_OPTS%
@endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%JVM_OPTIONS%" & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!%
if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
exit /b 1

2
distribution/src/bin/elasticsearch.bat
View File

@ -44,7 +44,7 @@ IF ERRORLEVEL 1 (
set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
@setlocal
for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" ^|^| echo jvm_options_parser_failed`) do set JVM_OPTIONS=%%a
@endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%JVM_OPTIONS%" & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!% %ES_JAVA_OPTS%
@endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%JVM_OPTIONS%" & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!%
if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
exit /b 1

113
distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java
View File

@ -19,24 +19,28 @@
package org.elasticsearch.tools.launchers;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Tunes Elasticsearch JVM settings based on inspection of provided JVM options.
*/
final class JvmErgonomics {
private static final long KB = 1024L;
private static final long MB = 1024L * 1024L;
private static final long GB = 1024L * 1024L * 1024L;
private JvmErgonomics() {
throw new AssertionError("No instances intended");
@ -48,48 +52,74 @@ final class JvmErgonomics {
* @param userDefinedJvmOptions A list of JVM options that have been defined by the user.
* @return A list of additional JVM options to set.
*/
static List<String> choose(List<String> userDefinedJvmOptions) {
List<String> ergonomicChoices = new ArrayList<>();
Long heapSize = extractHeapSize(userDefinedJvmOptions);
Map<String, String> systemProperties = extractSystemProperties(userDefinedJvmOptions);
if (heapSize != null) {
if (systemProperties.containsKey("io.netty.allocator.type") == false) {
if (heapSize <= 1 * GB) {
ergonomicChoices.add("-Dio.netty.allocator.type=unpooled");
} else {
ergonomicChoices.add("-Dio.netty.allocator.type=pooled");
}
static List<String> choose(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
final List<String> ergonomicChoices = new ArrayList<>();
final Map<String, Optional<String>> finalJvmOptions = finalJvmOptions(userDefinedJvmOptions);
final long heapSize = extractHeapSize(finalJvmOptions);
final Map<String, String> systemProperties = extractSystemProperties(userDefinedJvmOptions);
if (systemProperties.containsKey("io.netty.allocator.type") == false) {
if (heapSize <= 1 << 30) {
ergonomicChoices.add("-Dio.netty.allocator.type=unpooled");
} else {
ergonomicChoices.add("-Dio.netty.allocator.type=pooled");
}
}
return ergonomicChoices;
}
private static final Pattern MAX_HEAP_SIZE = Pattern.compile("^(-Xmx|-XX:MaxHeapSize=)(?<size>\\d+)(?<unit>\\w)?$");
private static final Pattern OPTION =
Pattern.compile("^\\s*\\S+\\s+(?<flag>\\S+)\\s+:?=\\s+(?<value>\\S+)?\\s+\\{[^}]+?\\}\\s+\\{[^}]+}");
static Map<String, Optional<String>> finalJvmOptions(
final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
return Collections.unmodifiableMap(flagsFinal(userDefinedJvmOptions).stream()
.map(OPTION::matcher).filter(Matcher::matches)
.collect(Collectors.toMap(m -> m.group("flag"), m -> Optional.ofNullable(m.group("value")))));
}
private static List<String> flagsFinal(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
/*
* To deduce the final set of JVM options that Elasticsearch is going to start with, we start a separate Java process with the JVM
* options that we would pass on the command line. For this Java process we will add two additional flags, -XX:+PrintFlagsFinal and
* -version. This causes the Java process that we start to parse the JVM options into their final values, display them on standard
* output, print the version to standard error, and then exit. The JVM itself never bootstraps, and therefore this process is
* lightweight. By doing this, we get the JVM options parsed exactly as the JVM that we are going to execute would parse them
* without having to implement our own JVM option parsing logic.
*/
final String java = Paths.get(System.getProperty("java.home"), "bin", "java").toString();
final List<String> command =
Collections.unmodifiableList(
Stream.of(Stream.of(java), userDefinedJvmOptions.stream(), Stream.of("-XX:+PrintFlagsFinal"), Stream.of("-version"))
.reduce(Stream::concat)
.get()
.collect(Collectors.toList()));
final Process process = new ProcessBuilder().command(command).start();
final List<String> output = readLinesFromInputStream(process.getInputStream());
final List<String> error = readLinesFromInputStream(process.getErrorStream());
final int status = process.waitFor();
if (status != 0) {
final String message = String.format(
Locale.ROOT,
"starting java failed with [%d]\noutput:\n%s\nerror:\n%s",
status,
String.join("\n", output),
String.join("\n", error));
throw new RuntimeException(message);
} else {
return output;
}
}
private static List<String> readLinesFromInputStream(final InputStream is) throws IOException {
try (InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
BufferedReader br = new BufferedReader(isr)) {
return Collections.unmodifiableList(br.lines().collect(Collectors.toList()));
}
}
// package private for testing
static Long extractHeapSize(List<String> userDefinedJvmOptions) {
for (String jvmOption : userDefinedJvmOptions) {
final Matcher matcher = MAX_HEAP_SIZE.matcher(jvmOption);
if (matcher.matches()) {
final long size = Long.parseLong(matcher.group("size"));
final String unit = matcher.group("unit");
if (unit == null) {
return size;
} else {
switch (unit.toLowerCase(Locale.ROOT)) {
case "k":
return size * KB;
case "m":
return size * MB;
case "g":
return size * GB;
default:
throw new IllegalArgumentException("Unknown unit [" + unit + "] for max heap size in [" + jvmOption + "]");
}
}
}
}
return null;
static Long extractHeapSize(final Map<String, Optional<String>> finalJvmOptions) {
return Long.parseLong(finalJvmOptions.get("MaxHeapSize").get());
}
private static final Pattern SYSTEM_PROPERTY = Pattern.compile("^-D(?<key>[\\w+].*?)=(?<value>.*)$");
@ -105,4 +135,5 @@ final class JvmErgonomics {
}
return systemProperties;
}
}

20
distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java
View File

@ -19,12 +19,14 @@
package org.elasticsearch.tools.launchers;
import org.elasticsearch.tools.java_version_checker.JavaVersion;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
@ -37,8 +39,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.elasticsearch.tools.java_version_checker.JavaVersion;
import java.util.stream.Collectors;
/**
* Parses JVM options from a file and prints a single line with all JVM options to standard output.
@ -51,14 +52,14 @@ final class JvmOptionsParser {
*
* @param args the args to the program which should consist of a single option, the path to the JVM options
*/
public static void main(final String[] args) throws IOException {
public static void main(final String[] args) throws InterruptedException, IOException {
if (args.length != 1) {
throw new IllegalArgumentException("expected one argument specifying path to jvm.options but was " + Arrays.toString(args));
}
final List<String> jvmOptions = new ArrayList<>();
final SortedMap<Integer, String> invalidLines = new TreeMap<>();
try (InputStream is = Files.newInputStream(Paths.get(args[0]));
Reader reader = new InputStreamReader(is, Charset.forName("UTF-8"));
Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
BufferedReader br = new BufferedReader(reader)) {
parse(
JavaVersion.majorVersion(JavaVersion.CURRENT),
@ -78,7 +79,14 @@ final class JvmOptionsParser {
}
if (invalidLines.isEmpty()) {
List<String> ergonomicJvmOptions = JvmErgonomics.choose(jvmOptions);
// now append the JVM options from ES_JAVA_OPTS
final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS");
if (environmentJvmOptions != null) {
jvmOptions.addAll(Arrays.stream(environmentJvmOptions.split("\\s+"))
.filter(s -> s.trim().isEmpty() == false)
.collect(Collectors.toList()));
}
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(jvmOptions);
jvmOptions.addAll(ergonomicJvmOptions);
final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(jvmOptions);
Launchers.outPrintln(spaceDelimitedJvmOptions);

67
distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java
View File

@ -19,38 +19,70 @@
package org.elasticsearch.tools.launchers;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.hasToString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class JvmErgonomicsTests extends LaunchersTestCase {
public void testExtractValidHeapSize() {
assertEquals(Long.valueOf(1024), JvmErgonomics.extractHeapSize(Collections.singletonList("-Xmx1024")));
assertEquals(Long.valueOf(2L * 1024 * 1024 * 1024), JvmErgonomics.extractHeapSize(Collections.singletonList("-Xmx2g")));
assertEquals(Long.valueOf(32 * 1024 * 1024), JvmErgonomics.extractHeapSize(Collections.singletonList("-Xmx32M")));
assertEquals(Long.valueOf(32 * 1024 * 1024), JvmErgonomics.extractHeapSize(Collections.singletonList("-XX:MaxHeapSize=32M")));
public void testExtractValidHeapSizeUsingXmx() throws InterruptedException, IOException {
assertThat(
JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx2g"))),
equalTo(2L << 30));
}
public void testExtractInvalidHeapSize() {
public void testExtractValidHeapSizeUsingMaxHeapSize() throws InterruptedException, IOException {
assertThat(
JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-XX:MaxHeapSize=2g"))),
equalTo(2L << 30));
}
public void testExtractValidHeapSizeNoOptionPresent() throws InterruptedException, IOException {
assertThat(
JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.emptyList())),
greaterThan(0L));
}
public void testHeapSizeInvalid() throws InterruptedException, IOException {
try {
JvmErgonomics.extractHeapSize(Collections.singletonList("-Xmx2T"));
fail("Expected IllegalArgumentException to be raised");
} catch (IllegalArgumentException expected) {
assertEquals("Unknown unit [T] for max heap size in [-Xmx2T]", expected.getMessage());
JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx2Z")));
fail("expected starting java to fail");
} catch (final RuntimeException e) {
assertThat(e, hasToString(containsString(("starting java failed"))));
assertThat(e, hasToString(containsString(("Invalid maximum heap size: -Xmx2Z"))));
}
}
public void testExtractNoHeapSize() {
assertNull("No spaces allowed", JvmErgonomics.extractHeapSize(Collections.singletonList("-Xmx 1024")));
assertNull("JVM option is not present", JvmErgonomics.extractHeapSize(Collections.singletonList("")));
assertNull("Multiple JVM options per line", JvmErgonomics.extractHeapSize(Collections.singletonList("-Xms2g -Xmx2g")));
public void testHeapSizeTooSmall() throws InterruptedException, IOException {
try {
JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx1024")));
fail("expected starting java to fail");
} catch (final RuntimeException e) {
assertThat(e, hasToString(containsString(("starting java failed"))));
assertThat(e, hasToString(containsString(("Too small maximum heap"))));
}
}
public void testHeapSizeWithSpace() throws InterruptedException, IOException {
try {
JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx 1024")));
fail("expected starting java to fail");
} catch (final RuntimeException e) {
assertThat(e, hasToString(containsString(("starting java failed"))));
assertThat(e, hasToString(containsString(("Invalid maximum heap size: -Xmx 1024"))));
}
}
public void testExtractSystemProperties() {
@ -69,15 +101,16 @@ public class JvmErgonomicsTests extends LaunchersTestCase {
assertTrue(parsedSystemProperties.isEmpty());
}
public void testLittleMemoryErgonomicChoices() {
public void testLittleMemoryErgonomicChoices() throws InterruptedException, IOException {
String smallHeap = randomFrom(Arrays.asList("64M", "512M", "1024M", "1G"));
List<String> expectedChoices = Collections.singletonList("-Dio.netty.allocator.type=unpooled");
assertEquals(expectedChoices, JvmErgonomics.choose(Arrays.asList("-Xms" + smallHeap, "-Xmx" + smallHeap)));
}
public void testPlentyMemoryErgonomicChoices() {
public void testPlentyMemoryErgonomicChoices() throws InterruptedException, IOException {
String largeHeap = randomFrom(Arrays.asList("1025M", "2048M", "2G", "8G"));
List<String> expectedChoices = Collections.singletonList("-Dio.netty.allocator.type=pooled");
assertEquals(expectedChoices, JvmErgonomics.choose(Arrays.asList("-Xms" + largeHeap, "-Xmx" + largeHeap)));
}
}

4
docs/reference/ilm/policy-definitions.asciidoc
View File

@ -84,6 +84,10 @@ executing.
The below list shows the actions which are available in each phase.
NOTE: The order that configured actions are performed in within each phase is
determined by automatically by {ilm-init}, and cannot be changed by changing the
policy definition.
* Hot
- <<ilm-set-priority-action,Set Priority>>
- <<ilm-rollover-action,Rollover>>

2
docs/reference/ingest/processors/geoip.asciidoc
View File

@ -27,7 +27,7 @@ uncompressed. The `ingest-geoip` config directory is located at `$ES_HOME/config
| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document
|======
*Depends on what is available in `database_field`:
*Depends on what is available in `database_file`:
* If the GeoLite2 City database is used, then the following fields may be added under the `target_field`: `ip`,
`country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `latitude`, `longitude`

2
gradle.properties
View File

@ -1,3 +1,3 @@
org.gradle.daemon=true
org.gradle.jvmargs=-Xmx2g -XX:+HeapDumpOnOutOfMemoryError
org.gradle.jvmargs=-Xmx2g -XX:+HeapDumpOnOutOfMemoryError -Xss2m
options.forkOptions.memoryMaximumSize=2g

4
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.3-all.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionSha256Sum=f4d820c2a9685710eba5b92f10e0e4fb20e0d6c0dd1f46971e658160f25e7147
distributionSha256Sum=14cd15fc8cc8705bd69dcfa3c8fefb27eb7027f5de4b47a8b279218f76895a91

6
qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java
View File

@ -56,6 +56,7 @@ import static org.elasticsearch.packaging.util.Packages.startElasticsearch;
import static org.elasticsearch.packaging.util.Packages.stopElasticsearch;
import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
import static org.elasticsearch.packaging.util.Platforms.getOsRelease;
import static org.elasticsearch.packaging.util.Platforms.isDPKG;
import static org.elasticsearch.packaging.util.Platforms.isSystemd;
import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
import static org.elasticsearch.packaging.util.ServerUtils.runElasticsearchTests;
@ -78,6 +79,11 @@ public abstract class PackageTestCase extends PackagingTestCase {
sh = newShell();
}
public void test05CheckLintian() throws Exception {
assumeTrue(isDPKG());
sh.run("lintian --fail-on-warnings " + FileUtils.getDistributionFile(distribution()));
}
public void test10InstallPackage() throws Exception {
assertRemoved(distribution());
installation = install(distribution());

4
server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
View File

@ -157,11 +157,11 @@ public class ReadOnlyEngine extends Engine {
protected final DirectoryReader wrapReader(DirectoryReader reader,
Function<DirectoryReader, DirectoryReader> readerWrapperFunction) throws IOException {
reader = ElasticsearchDirectoryReader.wrap(reader, engineConfig.getShardId());
if (engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
reader = new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
}
return readerWrapperFunction.apply(reader);
reader = readerWrapperFunction.apply(reader);
return ElasticsearchDirectoryReader.wrap(reader, engineConfig.getShardId());
}
protected DirectoryReader open(IndexCommit commit) throws IOException {

1
server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java
View File

@ -1065,6 +1065,7 @@ public class CoordinatorTests extends ESTestCase {
cluster1.stabilise();
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/41967")
public void testDiscoveryUsesNodesFromLastClusterState() {
final Cluster cluster = new Cluster(randomIntBetween(3, 5));
cluster.runRandomly();

13
server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java
View File

@ -18,9 +18,12 @@
*/
package org.elasticsearch.index.engine;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.Version;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.seqno.SeqNoStats;
@ -32,7 +35,9 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import static org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader.getElasticsearchDirectoryReader;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
public class ReadOnlyEngineTests extends EngineTestCase {
@ -80,6 +85,13 @@ public class ReadOnlyEngineTests extends EngineTestCase {
Engine.Searcher external = readOnlyEngine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL);
Engine.Searcher internal = readOnlyEngine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
assertSame(external.reader(), internal.reader());
assertThat(external.reader(), instanceOf(DirectoryReader.class));
DirectoryReader dirReader = external.getDirectoryReader();
ElasticsearchDirectoryReader esReader = getElasticsearchDirectoryReader(dirReader);
IndexReader.CacheHelper helper = esReader.getReaderCacheHelper();
assertNotNull(helper);
assertEquals(helper.getKey(), dirReader.getReaderCacheHelper().getKey());
IOUtils.close(external, internal);
// the locked down engine should still point to the previous commit
assertThat(readOnlyEngine.getLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
@ -88,7 +100,6 @@ public class ReadOnlyEngineTests extends EngineTestCase {
try (Engine.GetResult getResult = readOnlyEngine.get(get, readOnlyEngine::acquireSearcher)) {
assertTrue(getResult.exists());
}
}
// Close and reopen the main engine
try (InternalEngine recoveringEngine = new InternalEngine(config)) {