Merge branch 'master' into feature/query-refactoring

This commit is contained in:
Christoph Büscher 2015-07-28 11:28:45 +02:00
commit b780346cb7
192 changed files with 2040 additions and 759 deletions

View File

@ -44,7 +44,7 @@ In order to run Elasticsearch from source without building a package, you can
run it using Maven:
-------------------------------------
mvn compile exec:exec
mvn package -Drun -DskipTests
-------------------------------------
=== Test case filtering.
@ -318,4 +318,6 @@ environment before the tests are executed:
ES_CLEAN_BEFORE_TEST=true bats 30_deb_package.bats
---------------------------------------------------------------------------
The current mode of execution is to copy all the packages that should be tested
into one directory, then copy the bats files into the same directory and run
those.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -18,24 +18,9 @@
<properties>
<!-- Properties used for building RPM & DEB packages (see common/packaging.properties) -->
<packaging.elasticsearch.home.dir>/usr/share/elasticsearch</packaging.elasticsearch.home.dir>
<packaging.elasticsearch.bin.dir>/usr/share/elasticsearch/bin</packaging.elasticsearch.bin.dir>
<packaging.elasticsearch.conf.dir>/etc/elasticsearch</packaging.elasticsearch.conf.dir>
<packaging.elasticsearch.data.dir>/var/lib/elasticsearch</packaging.elasticsearch.data.dir>
<packaging.elasticsearch.user>elasticsearch</packaging.elasticsearch.user>
<packaging.elasticsearch.group>elasticsearch</packaging.elasticsearch.group>
<packaging.elasticsearch.log.dir>/var/log/elasticsearch</packaging.elasticsearch.log.dir>
<packaging.elasticsearch.plugins.dir>${packaging.elasticsearch.home.dir}/plugins</packaging.elasticsearch.plugins.dir>
<packaging.elasticsearch.pid.dir>/var/run/elasticsearch</packaging.elasticsearch.pid.dir>
<packaging.elasticsearch.systemd.dir>/usr/lib/systemd/system</packaging.elasticsearch.systemd.dir>
<packaging.elasticsearch.systemd.sysctl.dir>/usr/lib/sysctl.d</packaging.elasticsearch.systemd.sysctl.dir>
<packaging.elasticsearch.tmpfilesd.dir>/usr/lib/tmpfiles.d</packaging.elasticsearch.tmpfilesd.dir>
<deb.sign>false</deb.sign>
<deb.sign.method>dpkg-sig</deb.sign.method>
<skip.integ.tests>true</skip.integ.tests>
</properties>
<dependencies>
<dependency>
<groupId>org.hamcrest</groupId>
@ -226,27 +211,10 @@
<version>1.1.0</version>
</dependency>
<!-- We don't use this since the publish pom is then messed up -->
<!--
<dependency>
<groupId>sigar</groupId>
<artifactId>sigar</artifactId>
<scope>system</scope>
<systemPath>${project.basedir}/lib/sigar/sigar-1.6.4.jar</systemPath>
<optional>true</optional>
</dependency>
-->
</dependencies>
<build>
<!-- This file contains all the common properties used to build
the different packages (tar.gz, deb, rpm) using Maven resources plugin -->
<filters>
<filter>${project.basedir}/src/packaging/common/packaging.properties</filter>
</filters>
<resources>
<resource>
<directory>${project.basedir}/src/main/java</directory>
@ -442,532 +410,6 @@
</filters>
</configuration>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>2.6</version>
<executions>
<execution>
<id>copy-resources</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/bin</outputDirectory>
<resources>
<resource>
<directory>${project.basedir}/bin</directory>
<filtering>true</filtering>
<excludes>
<exclude>*.exe</exclude>
</excludes>
</resource>
<resource>
<directory>${project.basedir}/bin</directory>
<filtering>false</filtering>
<includes>
<include>*.exe</include>
</includes>
</resource>
</resources>
</configuration>
</execution>
<!-- Filters some files and uses packaging.properties when building the .deb package -->
<execution>
<id>copy-resources-deb</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/generated-packaging/deb/</outputDirectory>
<filters>
<filter>${project.basedir}/src/packaging/common/packaging.properties</filter>
<filter>${project.basedir}/src/packaging/deb/packaging.properties</filter>
</filters>
<resources>
<resource>
<directory>${project.basedir}/src/packaging/common/</directory>
<filtering>true</filtering>
<includes>
<include>**/*</include>
</includes>
<excludes>
<exclude>packaging.properties</exclude>
</excludes>
</resource>
<resource>
<directory>${project.basedir}/src/packaging/deb/</directory>
<filtering>true</filtering>
<includes>
<include>**/*</include>
</includes>
<excludes>
<exclude>packaging.properties</exclude>
</excludes>
</resource>
<resource>
<directory>${project.basedir}</directory>
<filtering>true</filtering>
<includes>
<include>bin/elasticsearch</include>
<include>bin/elasticsearch.in.sh</include>
<include>bin/plugin</include>
</includes>
</resource>
</resources>
</configuration>
</execution>
<!-- Filters some files and uses packaging.properties when building the .rpm package -->
<execution>
<id>copy-resources-rpm</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/generated-packaging/rpm/</outputDirectory>
<filters>
<filter>${project.basedir}/src/packaging/common/packaging.properties</filter>
<filter>${project.basedir}/src/packaging/rpm/packaging.properties</filter>
</filters>
<resources>
<resource>
<directory>src/packaging/common/</directory>
<filtering>true</filtering>
<includes>
<include>**/*</include>
</includes>
<excludes>
<exclude>packaging.properties</exclude>
</excludes>
</resource>
<resource>
<directory>src/packaging/rpm/</directory>
<filtering>true</filtering>
<includes>
<include>**/*</include>
</includes>
<excludes>
<exclude>packaging.properties</exclude>
</excludes>
</resource>
<resource>
<directory>${project.basedir}</directory>
<filtering>true</filtering>
<includes>
<include>bin/elasticsearch</include>
<include>bin/elasticsearch.in.sh</include>
<include>bin/plugin</include>
</includes>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<outputDirectory>${project.build.directory}/releases/</outputDirectory>
<descriptors>
<descriptor>${project.basedir}/src/main/assemblies/targz-bin.xml</descriptor>
<descriptor>${project.basedir}/src/main/assemblies/zip-bin.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
<!-- Deb Packaging -->
<plugin>
<!-- some infos https://github.com/tcurdt/jdeb/blob/master/docs/maven.md -->
<artifactId>jdeb</artifactId>
<groupId>org.vafer</groupId>
<version>1.4</version>
<configuration>
<deb>${project.build.directory}/releases/${project.artifactId}-${project.version}.deb</deb>
<controlDir>${project.build.directory}/generated-packaging/deb/scripts</controlDir>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>jdeb</goal>
</goals>
<configuration>
<signPackage>${deb.sign}</signPackage>
<keyring>${gpg.keyring}</keyring>
<key>${gpg.key}</key>
<passphrase>${gpg.passphrase}</passphrase>
<signMethod>${deb.sign.method}</signMethod>
<dataSet>
<!-- Add bin directory -->
<data>
<src>${project.build.directory}/generated-packaging/deb/bin</src>
<type>directory</type>
<includes>elasticsearch,elasticsearch.in.sh,plugin</includes>
<mapper>
<type>perm</type>
<prefix>${packaging.elasticsearch.bin.dir}</prefix>
<filemode>755</filemode>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add configuration files -->
<data>
<src>${project.basedir}/config</src>
<type>directory</type>
<excludes>.DS_Store</excludes>
<mapper>
<type>perm</type>
<prefix>${packaging.elasticsearch.conf.dir}</prefix>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add environment vars file -->
<data>
<src>${project.build.directory}/generated-packaging/deb/env/elasticsearch</src>
<type>file</type>
<mapper>
<type>perm</type>
<prefix>/etc/default</prefix>
<filemode>644</filemode>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add libs -->
<data>
<src>${project.build.directory}/</src>
<includes>${project.build.finalName}.jar</includes>
<type>directory</type>
<mapper>
<type>perm</type>
<prefix>${packaging.elasticsearch.home.dir}/lib</prefix>
<user>root</user>
<group>root</group>
</mapper>
</data>
<data>
<src>${project.build.directory}/lib</src>
<excludes>${project.build.finalName}-shaded.jar,${project.build.finalName}-sources.jar,${project.build.finalName}-tests.jar,${project.build.finalName}-test-sources.jar,slf4j-api-*.jar</excludes>
<type>directory</type>
<mapper>
<type>perm</type>
<prefix>${packaging.elasticsearch.home.dir}/lib</prefix>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add init.d files -->
<data>
<src>${project.build.directory}/generated-packaging/deb/init.d/</src>
<type>directory</type>
<excludes>.DS_Store</excludes>
<mapper>
<type>perm</type>
<prefix>/etc/init.d</prefix>
<filemode>755</filemode>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Adds systemd file -->
<data>
<src>${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.service</src>
<dst>${packaging.elasticsearch.systemd.dir}/elasticsearch.service</dst>
<type>file</type>
</data>
<!-- Adds systemd/sysctl.d configuration file -->
<data>
<src>${project.build.directory}/generated-packaging/deb/systemd/sysctl/elasticsearch.conf</src>
<dst>${packaging.elasticsearch.systemd.sysctl.dir}/elasticsearch.conf</dst>
<type>file</type>
</data>
<!-- Adds systemd/tmpfiles.d configuration file -->
<data>
<src>${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.conf</src>
<dst>${packaging.elasticsearch.tmpfilesd.dir}/elasticsearch.conf</dst>
<type>file</type>
</data>
<!-- Add lintian files -->
<data>
<src>${project.build.directory}/generated-packaging/deb/lintian</src>
<type>directory</type>
<excludes>.DS_Store</excludes>
<mapper>
<type>perm</type>
<prefix>/usr/share/lintian/overrides</prefix>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add readme files -->
<data>
<src>${project.basedir}/</src>
<includes>*.txt, *.textile</includes>
<excludes>LICENSE.txt, .DS_Store</excludes>
<type>directory</type>
<mapper>
<type>perm</type>
<prefix>${packaging.elasticsearch.home.dir}</prefix>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add license files -->
<data>
<src>${project.build.directory}/generated-packaging/deb/copyright</src>
<dst>/usr/share/doc/elasticsearch/copyright</dst>
<type>file</type>
</data>
<!-- Adds and sets permission on default directories -->
<data>
<type>template</type>
<paths>
<path>${packaging.elasticsearch.data.dir}</path>
<path>${packaging.elasticsearch.log.dir}</path>
<path>${packaging.elasticsearch.plugins.dir}</path>
<path>${packaging.elasticsearch.pid.dir}</path>
</paths>
<mapper>
<type>perm</type>
<dirmode>755</dirmode>
<user>${packaging.elasticsearch.user}</user>
<group>${packaging.elasticsearch.group}</group>
</mapper>
</data>
</dataSet>
</configuration>
</execution>
</executions>
</plugin>
<!-- Rpm Packaging -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<version>2.1.3</version>
<configuration>
<distribution>Elasticsearch</distribution>
<group>Application/Internet</group>
<packager>Elasticsearch</packager>
<prefix>/usr</prefix>
<changelogFile>src/changelog</changelogFile>
<defineStatements>
<defineStatement>_unpackaged_files_terminate_build 0</defineStatement>
<defineStatement>_binaries_in_noarch_packages_terminate_build 0</defineStatement>
</defineStatements>
<defaultFilemode>644</defaultFilemode>
<defaultDirmode>755</defaultDirmode>
<defaultUsername>root</defaultUsername>
<defaultGroupname>root</defaultGroupname>
<mappings>
<!-- Add bin directory -->
<mapping>
<directory>${packaging.elasticsearch.bin.dir}/</directory>
<filemode>755</filemode>
<sources>
<source>
<location>${project.build.directory}/generated-packaging/rpm/bin</location>
<includes>
<include>elasticsearch</include>
<include>elasticsearch.in.sh</include>
<include>plugin</include>
</includes>
</source>
</sources>
</mapping>
<!-- Add configuration files -->
<mapping>
<directory>${packaging.elasticsearch.conf.dir}/</directory>
<configuration>noreplace</configuration>
<sources>
<source>
<location>${project.basedir}/config/</location>
<includes>
<include>*.yml</include>
</includes>
</source>
</sources>
</mapping>
<!-- Add environment vars file -->
<mapping>
<directory>/etc/sysconfig/</directory>
<directoryIncluded>false</directoryIncluded>
<configuration>noreplace</configuration>
<sources>
<source>
<location>${project.build.directory}/generated-packaging/rpm/env/</location>
<includes>
<include>elasticsearch</include>
</includes>
</source>
</sources>
</mapping>
<!-- Add libs -->
<mapping>
<directory>${packaging.elasticsearch.home.dir}/lib</directory>
<sources>
<source>
<location>target/lib/</location>
<excludes>
<exclude>${project.build.finalName}-shaded.jar</exclude>
<exclude>${project.build.finalName}-sources.jar</exclude>
<exclude>${project.build.finalName}-tests.jar</exclude>
<exclude>${project.build.finalName}-test-sources.jar</exclude>
<exclude>slf4j-api-*.jar</exclude>
</excludes>
</source>
<source>
<location>${project.build.directory}/</location>
<includes>
<include>${project.build.finalName}.jar</include>
</includes>
</source>
</sources>
</mapping>
<!-- Add init.d files -->
<mapping>
<directory>/etc/init.d</directory>
<directoryIncluded>false</directoryIncluded>
<filemode>755</filemode>
<configuration>true</configuration>
<sources>
<source>
<location>${project.build.directory}/generated-packaging/rpm/init.d</location>
<includes>
<include>elasticsearch</include>
</includes>
</source>
</sources>
</mapping>
<!-- Adds systemd file -->
<mapping>
<directory>${packaging.elasticsearch.systemd.dir}</directory>
<directoryIncluded>false</directoryIncluded>
<configuration>true</configuration>
<sources>
<source>
<location>${project.build.directory}/generated-packaging/rpm/systemd</location>
<includes>
<include>elasticsearch.service</include>
</includes>
</source>
</sources>
</mapping>
<!-- Adds systemd/sysctl.d configuration file -->
<mapping>
<directory>${packaging.elasticsearch.systemd.sysctl.dir}</directory>
<configuration>true</configuration>
<sources>
<source>
<location>${project.build.directory}/generated-packaging/rpm/systemd/sysctl</location>
<includes>
<include>elasticsearch.conf</include>
</includes>
</source>
</sources>
</mapping>
<!-- Adds systemd/tmpfiles.d configuration file -->
<mapping>
<directory>${packaging.elasticsearch.tmpfilesd.dir}</directory>
<configuration>true</configuration>
<sources>
<source>
<location>${project.build.directory}/generated-packaging/rpm/systemd/</location>
<includes>
<include>elasticsearch.conf</include>
</includes>
</source>
</sources>
</mapping>
<!-- Add readme files -->
<mapping>
<directory>${packaging.elasticsearch.home.dir}</directory>
<sources>
<source>
<location>.</location>
<includes>
<include>LICENSE.txt</include>
<include>NOTICE.txt</include>
<include>README.textile</include>
</includes>
</source>
</sources>
</mapping>
<!-- Add and sets permission on default directories -->
<mapping>
<directory>${packaging.elasticsearch.data.dir}</directory>
<filemode>755</filemode>
<username>${packaging.elasticsearch.user}</username>
<groupname>${packaging.elasticsearch.group}</groupname>
</mapping>
<mapping>
<directory>${packaging.elasticsearch.log.dir}</directory>
<filemode>755</filemode>
<username>${packaging.elasticsearch.user}</username>
<groupname>${packaging.elasticsearch.group}</groupname>
</mapping>
<mapping>
<directory>${packaging.elasticsearch.plugins.dir}</directory>
<filemode>755</filemode>
<username>${packaging.elasticsearch.user}</username>
<groupname>${packaging.elasticsearch.group}</groupname>
</mapping>
<mapping>
<directory>${packaging.elasticsearch.pid.dir}</directory>
<filemode>755</filemode>
<username>${packaging.elasticsearch.user}</username>
<groupname>${packaging.elasticsearch.group}</groupname>
</mapping>
</mappings>
<preinstallScriptlet>
<scriptFile>${project.build.directory}/generated-packaging/rpm/scripts/preinst</scriptFile>
<fileEncoding>utf-8</fileEncoding>
</preinstallScriptlet>
<postinstallScriptlet>
<scriptFile>${project.build.directory}/generated-packaging/rpm/scripts/postinst</scriptFile>
<fileEncoding>utf-8</fileEncoding>
</postinstallScriptlet>
<preremoveScriptlet>
<scriptFile>${project.build.directory}/generated-packaging/rpm/scripts/prerm</scriptFile>
<fileEncoding>utf-8</fileEncoding>
</preremoveScriptlet>
<postremoveScriptlet>
<scriptFile>${project.build.directory}/generated-packaging/rpm/scripts/postrm</scriptFile>
<fileEncoding>utf-8</fileEncoding>
</postremoveScriptlet>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@ -1014,32 +456,6 @@
</target>
</configuration>
</execution>
<!-- start up external cluster -->
<execution>
<id>integ-setup</id>
<phase>pre-integration-test</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<ant antfile="${elasticsearch.integ.antfile}" target="start-external-cluster"/>
</target>
</configuration>
</execution>
<!-- shut down external cluster -->
<execution>
<id>integ-teardown</id>
<phase>post-integration-test</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<ant antfile="${elasticsearch.integ.antfile}" target="stop-external-cluster"/>
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
@ -1085,54 +501,5 @@
</activation>
<!-- not including license-maven-plugin is sufficent to expose default license -->
</profile>
<profile>
<id>release</id> <!-- named after the parents release profile to be activated -->
<activation>
<property>
<name>package.rpm</name>
<value>true</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<executions>
<execution>
<id>attach-rpm</id>
<goals>
<goal>attached-rpm</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>sign-rpm</id>
<activation>
<property>
<name>rpm.sign</name>
<value>true</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<configuration>
<keyname>${gpg.key}</keyname>
<keypath>${gpg.keyring}</keypath>
<keyPassphrase>
<passphrase>${gpg.passphrase}</passphrase>
</keyPassphrase>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

View File

@ -395,7 +395,6 @@ public class ClusterStatsNodes implements ToXContent, Streamable {
}
count++;
if (nodeStats.getProcess().getCpu() != null) {
// with no sigar, this may not be available
cpuPercent += nodeStats.getProcess().getCpu().getPercent();
}
long fd = nodeStats.getProcess().getOpenFileDescriptors();

View File

@ -130,8 +130,8 @@ public class MultiSearchRequest extends ActionRequest<MultiSearchRequest> implem
searchRequest.types(nodeStringArrayValue(value));
} else if ("search_type".equals(entry.getKey()) || "searchType".equals(entry.getKey())) {
searchRequest.searchType(nodeStringValue(value, null));
} else if ("query_cache".equals(entry.getKey()) || "queryCache".equals(entry.getKey())) {
searchRequest.queryCache(nodeBooleanValue(value));
} else if ("request_cache".equals(entry.getKey()) || "requestCache".equals(entry.getKey())) {
searchRequest.requestCache(nodeBooleanValue(value));
} else if ("preference".equals(entry.getKey())) {
searchRequest.preference(nodeStringValue(value, null));
} else if ("routing".equals(entry.getKey())) {

View File

@ -529,12 +529,12 @@ public class SearchRequest extends ActionRequest<SearchRequest> implements Indic
}
/**
* Sets if this request should use the query cache or not, assuming that it can (for
* Sets if this request should use the request cache or not, assuming that it can (for
* example, if "now" is used, it will never be cached). By default (not set, or null,
* will default to the index level setting if query cache is enabled or not).
* will default to the index level setting if request cache is enabled or not).
*/
public SearchRequest queryCache(Boolean queryCache) {
this.requestCache = queryCache;
public SearchRequest requestCache(Boolean requestCache) {
this.requestCache = requestCache;
return this;
}

View File

@ -957,12 +957,12 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
}
/**
* Sets if this request should use the query cache or not, assuming that it can (for
* Sets if this request should use the request cache or not, assuming that it can (for
* example, if "now" is used, it will never be cached). By default (not set, or null,
* will default to the index level setting if query cache is enabled or not).
* will default to the index level setting if request cache is enabled or not).
*/
public SearchRequestBuilder setQueryCache(Boolean queryCache) {
request.queryCache(queryCache);
public SearchRequestBuilder setRequestCache(Boolean requestCache) {
request.requestCache(requestCache);
return this;
}

View File

@ -49,6 +49,7 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
@ -465,7 +466,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(mappingsDir)) {
for (Path mappingFile : stream) {
final String fileName = mappingFile.getFileName().toString();
if (Files.isHidden(mappingFile)) {
if (FileSystemUtils.isHidden(mappingFile)) {
continue;
}
int lastDotIndex = fileName.lastIndexOf('.');

View File

@ -20,6 +20,7 @@
package org.elasticsearch.common.io;
import com.google.common.collect.Iterators;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.common.logging.ESLogger;
@ -84,6 +85,20 @@ public final class FileSystemUtils {
return false;
}
/**
* Check whether the file denoted by the given path is hidden.
* In practice, this will check if the file name starts with a dot.
* This should be preferred to {@link Files#isHidden(Path)} as this
* does not depend on the operating system.
*/
public static boolean isHidden(Path path) {
Path fileName = path.getFileName();
if (fileName == null) {
return false;
}
return fileName.toString().startsWith(".");
}
/**
* Appends the path to the given base and strips N elements off the path if strip is > 0.
*/
@ -334,4 +349,5 @@ public final class FileSystemUtils {
return Iterators.toArray(stream.iterator(), Path.class);
}
}
}

View File

@ -22,6 +22,7 @@ package org.elasticsearch.env;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.store.*;
@ -29,6 +30,7 @@ import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
@ -661,6 +663,56 @@ public class NodeEnvironment extends AbstractComponent implements Closeable {
return indices;
}
/**
* Tries to find all allocated shards for the given index
* on the current node. NOTE: This methods is prone to race-conditions on the filesystem layer since it might not
* see directories created concurrently or while it's traversing.
* @param index the index to filter shards
* @return a set of shard IDs
* @throws IOException if an IOException occurs
*/
public Set<ShardId> findAllShardIds(final Index index) throws IOException {
assert index != null;
if (nodePaths == null || locks == null) {
throw new IllegalStateException("node is not configured to store local location");
}
assert assertEnvIsLocked();
final Set<ShardId> shardIds = Sets.newHashSet();
String indexName = index.name();
for (final NodePath nodePath : nodePaths) {
Path location = nodePath.indicesPath;
if (Files.isDirectory(location)) {
try (DirectoryStream<Path> indexStream = Files.newDirectoryStream(location)) {
for (Path indexPath : indexStream) {
if (indexName.equals(indexPath.getFileName().toString())) {
shardIds.addAll(findAllShardsForIndex(indexPath));
}
}
}
}
}
return shardIds;
}
private static Set<ShardId> findAllShardsForIndex(Path indexPath) throws IOException {
Set<ShardId> shardIds = new HashSet<>();
if (Files.isDirectory(indexPath)) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexPath)) {
String currentIndex = indexPath.getFileName().toString();
for (Path shardPath : stream) {
if (Files.isDirectory(shardPath)) {
Integer shardId = Ints.tryParse(shardPath.getFileName().toString());
if (shardId != null) {
ShardId id = new ShardId(currentIndex, shardId);
shardIds.add(id);
}
}
}
}
}
return shardIds;
}
@Override
public void close() {
if (closed.compareAndSet(false, true) && locks != null) {

View File

@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.node.service.NodeService;
@ -183,7 +184,7 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> {
Path file = siteFile.resolve(sitePath);
// return not found instead of forbidden to prevent malicious requests to find out if files exist or dont exist
if (!Files.exists(file) || Files.isHidden(file) || !file.toAbsolutePath().normalize().startsWith(siteFile.toAbsolutePath().normalize())) {
if (!Files.exists(file) || FileSystemUtils.isHidden(file) || !file.toAbsolutePath().normalize().startsWith(siteFile.toAbsolutePath().normalize())) {
channel.sendResponse(new BytesRestResponse(NOT_FOUND));
return;
}
@ -197,7 +198,7 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> {
}
// We don't serve dir but if index.html exists in dir we should serve it
file = file.resolve("index.html");
if (!Files.exists(file) || Files.isHidden(file) || !Files.isRegularFile(file)) {
if (!Files.exists(file) || FileSystemUtils.isHidden(file) || !Files.isRegularFile(file)) {
channel.sendResponse(new BytesRestResponse(FORBIDDEN));
return;
}

View File

@ -93,7 +93,7 @@ public class RequestCacheStats implements Streamable, ToXContent {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.QUERY_CACHE_STATS);
builder.startObject(Fields.REQUEST_CACHE_STATS);
builder.byteSizeField(Fields.MEMORY_SIZE_IN_BYTES, Fields.MEMORY_SIZE, memorySize);
builder.field(Fields.EVICTIONS, getEvictions());
builder.field(Fields.HIT_COUNT, getHitCount());
@ -103,7 +103,7 @@ public class RequestCacheStats implements Streamable, ToXContent {
}
static final class Fields {
static final XContentBuilderString QUERY_CACHE_STATS = new XContentBuilderString("request_cache");
static final XContentBuilderString REQUEST_CACHE_STATS = new XContentBuilderString("request_cache");
static final XContentBuilderString MEMORY_SIZE = new XContentBuilderString("memory_size");
static final XContentBuilderString MEMORY_SIZE_IN_BYTES = new XContentBuilderString("memory_size_in_bytes");
static final XContentBuilderString EVICTIONS = new XContentBuilderString("evictions");

View File

@ -524,18 +524,38 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
* This method deletes the shard contents on disk for the given shard ID. This method will fail if the shard deleting
* is prevented by {@link #canDeleteShardContent(org.elasticsearch.index.shard.ShardId, org.elasticsearch.cluster.metadata.IndexMetaData)}
* of if the shards lock can not be acquired.
*
* On data nodes, if the deleted shard is the last shard folder in its index, the method will attempt to remove the index folder as well.
*
* @param reason the reason for the shard deletion
* @param shardId the shards ID to delete
* @param metaData the shards index metadata. This is required to access the indexes settings etc.
* @param clusterState . This is required to access the indexes settings etc.
* @throws IOException if an IOException occurs
*/
public void deleteShardStore(String reason, ShardId shardId, IndexMetaData metaData) throws IOException {
public void deleteShardStore(String reason, ShardId shardId, ClusterState clusterState) throws IOException {
final IndexMetaData metaData = clusterState.getMetaData().indices().get(shardId.getIndex());
final Settings indexSettings = buildIndexSettings(metaData);
if (canDeleteShardContent(shardId, indexSettings) == false) {
throw new IllegalStateException("Can't delete shard " + shardId);
}
nodeEnv.deleteShardDirectorySafe(shardId, indexSettings);
logger.trace("{} deleting shard reason [{}]", shardId, reason);
logger.debug("{} deleted shard reason [{}]", shardId, reason);
if (clusterState.nodes().localNode().isMasterNode() == false && // master nodes keep the index meta data, even if having no shards..
canDeleteIndexContents(shardId.index(), indexSettings)) {
if (nodeEnv.findAllShardIds(shardId.index()).isEmpty()) {
try {
// note that deleteIndexStore have more safety checks and may throw an exception if index was concurrently created.
deleteIndexStore("no longer used", metaData, clusterState);
} catch (Exception e) {
// wrap the exception to indicate we already deleted the shard
throw new ElasticsearchException("failed to delete unused index after deleting its last shard (" + shardId + ")", e);
}
} else {
logger.trace("[{}] still has shard stores, leaving as is", shardId.index());
}
}
}
/**

View File

@ -21,7 +21,6 @@ package org.elasticsearch.indices.store;
import org.apache.lucene.store.StoreRateLimiting;
import org.elasticsearch.cluster.*;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@ -288,28 +287,18 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
return;
}
clusterService.submitStateUpdateTask("indices_store", new ClusterStateNonMasterUpdateTask() {
clusterService.submitStateUpdateTask("indices_store ([" + shardId + "] active fully on other nodes)", new ClusterStateNonMasterUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
if (clusterState.getVersion() != currentState.getVersion()) {
logger.trace("not deleting shard {}, the update task state version[{}] is not equal to cluster state before shard active api call [{}]", shardId, currentState.getVersion(), clusterState.getVersion());
return currentState;
}
IndexMetaData indexMeta = clusterState.getMetaData().indices().get(shardId.getIndex());
try {
indicesService.deleteShardStore("no longer used", shardId, indexMeta);
indicesService.deleteShardStore("no longer used", shardId, currentState);
} catch (Throwable ex) {
logger.debug("{} failed to delete unallocated shard, ignoring", ex, shardId);
}
// if the index doesn't exists anymore, delete its store as well, but only if its a non master node, since master
// nodes keep the index metadata around
if (indicesService.hasIndex(shardId.getIndex()) == false && currentState.nodes().localNode().masterNode() == false) {
try {
indicesService.deleteIndexStore("no longer used", indexMeta, currentState);
} catch (Throwable ex) {
logger.debug("{} failed to delete unallocated index, ignoring", ex, shardId.getIndex());
}
}
return currentState;
}

View File

@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo;
import org.elasticsearch.bootstrap.Bootstrap;
import org.elasticsearch.bootstrap.JarHell;
@ -33,6 +32,7 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
@ -40,7 +40,6 @@ import org.elasticsearch.env.Environment;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
@ -54,7 +53,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory;
@ -312,7 +310,7 @@ public class PluginsService extends AbstractComponent {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsDirectory)) {
for (Path plugin : stream) {
try {
if (Files.isHidden(plugin)) {
if (FileSystemUtils.isHidden(plugin)) {
logger.trace("--- skip hidden plugin file[{}]", plugin.toAbsolutePath());
continue;
}

View File

@ -561,7 +561,9 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
}
bRef = bStream.bytes();
}
snapshotsBlobContainer.deleteBlob(SNAPSHOTS_FILE);
if (snapshotsBlobContainer.blobExists(SNAPSHOTS_FILE)) {
snapshotsBlobContainer.deleteBlob(SNAPSHOTS_FILE);
}
try (OutputStream output = snapshotsBlobContainer.createOutput(SNAPSHOTS_FILE)) {
bRef.writeTo(output);
}

View File

@ -61,7 +61,7 @@ public class RestPutWarmerAction extends BaseRestHandler {
PutWarmerRequest putWarmerRequest = new PutWarmerRequest(request.param("name"));
SearchRequest searchRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")))
.types(Strings.splitStringByCommaToArray(request.param("type")))
.queryCache(request.paramAsBoolean("query_cache", null))
.requestCache(request.paramAsBoolean("request_cache", null))
.source(request.content());
searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
putWarmerRequest.searchRequest(searchRequest);

View File

@ -110,7 +110,7 @@ public class RestSearchAction extends BaseRestHandler {
}
searchRequest.extraSource(parseSearchSource(request));
searchRequest.queryCache(request.paramAsBoolean("query_cache", null));
searchRequest.requestCache(request.paramAsBoolean("request_cache", null));
String scroll = request.param("scroll");
if (scroll != null) {

View File

@ -19,6 +19,8 @@
package org.elasticsearch.script.groovy;
import com.google.common.base.Charsets;
import com.google.common.hash.Hashing;
import groovy.lang.Binding;
import groovy.lang.GroovyClassLoader;
import groovy.lang.Script;
@ -49,7 +51,6 @@ import java.io.IOException;
import java.math.BigDecimal;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
/**
* Provides the infrastructure for Groovy as a scripting language for Elasticsearch
@ -57,7 +58,6 @@ import java.util.concurrent.atomic.AtomicLong;
public class GroovyScriptEngineService extends AbstractComponent implements ScriptEngineService {
public static final String NAME = "groovy";
private final AtomicLong counter = new AtomicLong();
private final GroovyClassLoader loader;
@Inject
@ -111,7 +111,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
@Override
public Object compile(String script) {
try {
return loader.parseClass(script, generateScriptName());
return loader.parseClass(script, Hashing.sha1().hashString(script, Charsets.UTF_8).toString());
} catch (Throwable e) {
if (logger.isTraceEnabled()) {
logger.trace("exception compiling Groovy script:", e);
@ -190,10 +190,6 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
return value;
}
private String generateScriptName() {
return "Script" + counter.incrementAndGet() + ".groovy";
}
public static final class GroovyScript implements ExecutableScript, LeafSearchScript {
private final CompiledScript compiledScript;

View File

@ -214,7 +214,7 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
source = new BytesArray(parser.binaryValue());
}
} else if (token.isValue()) {
if ("queryCache".equals(currentFieldName) || "query_cache".equals(currentFieldName)) {
if ("requestCache".equals(currentFieldName) || "request_cache".equals(currentFieldName)) {
queryCache = parser.booleanValue();
}
}
@ -242,7 +242,7 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
builder.startObject(entry.name(), XContentBuilder.FieldCaseConversion.NONE);
builder.field("types", entry.types());
if (entry.requestCache() != null) {
builder.field("queryCache", entry.requestCache());
builder.field("requestCache", entry.requestCache());
}
builder.field("source");
if (binary) {

View File

@ -1,4 +1,4 @@
{"index":["test0", "test1"], "query_cache": true}
{"index":["test0", "test1"], "request_cache": true}
{"query" : {"match_all" {}}}
{"index" : "test2,test3", "type" : "type1", "preference": "_local"}
{"query" : {"match_all" {}}}

View File

@ -32,6 +32,7 @@ import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists;
@ -173,4 +174,39 @@ public class FileSystemUtilsTests extends ElasticsearchTestCase {
assertEquals(FileSystemUtils.append(PathUtils.get("/foo/bar"), PathUtils.get("/hello/world/this_is/awesome"), 1),
PathUtils.get("/foo/bar/world/this_is/awesome"));
}
public void testIsHidden() {
for (String p : Arrays.asList(
"/",
"foo",
"/foo",
"foo.bar",
"/foo.bar",
"foo/bar",
"foo/./bar",
"foo/../bar",
"/foo/./bar",
"/foo/../bar"
)) {
Path path = PathUtils.get(p);
assertFalse(FileSystemUtils.isHidden(path));
}
for (String p : Arrays.asList(
".hidden",
".hidden.ext",
"/.hidden",
"/.hidden.ext",
"foo/.hidden",
"foo/.hidden.ext",
"/foo/.hidden",
"/foo/.hidden.ext",
".",
"..",
"foo/.",
"foo/.."
)) {
Path path = PathUtils.get(p);
assertTrue(FileSystemUtils.isHidden(path));
}
}
}

View File

@ -259,10 +259,10 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest {
// test explicit request parameter
assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setQueryCache(false).get().getHits().getTotalHits(), equalTo((long) numDocs));
assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(false).get().getHits().getTotalHits(), equalTo((long) numDocs));
assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0l));
assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setQueryCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs));
assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs));
assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0l));
// set the index level setting to false, and see that the reverse works
@ -273,7 +273,7 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest {
assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get().getHits().getTotalHits(), equalTo((long) numDocs));
assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0l));
assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setQueryCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs));
assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs));
assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0l));
}

View File

@ -21,6 +21,7 @@ package org.elasticsearch.indices.store;
import com.google.common.base.Predicate;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
@ -30,6 +31,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.*;
import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.Settings;
@ -37,12 +39,14 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.recovery.RecoverySource;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.elasticsearch.test.InternalTestCluster;
import org.elasticsearch.test.disruption.BlockClusterStateProcessing;
import org.elasticsearch.test.disruption.SingleNodeDisruption;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.test.transport.MockTransportService;
import org.elasticsearch.transport.TransportModule;
import org.elasticsearch.transport.TransportRequestOptions;
@ -55,6 +59,7 @@ import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import static java.lang.Thread.sleep;
@ -217,6 +222,87 @@ public class IndicesStoreIntegrationTests extends ElasticsearchIntegrationTest {
assertThat(waitForShardDeletion(node_4, "test", 0), equalTo(false));
}
@Test
@TestLogging("cluster.service:TRACE")
public void testShardActiveElsewhereDoesNotDeleteAnother() throws Exception {
Future<String> masterFuture = internalCluster().startNodeAsync(
Settings.builder().put("node.master", true, "node.data", false).build());
Future<List<String>> nodesFutures = internalCluster().startNodesAsync(4,
Settings.builder().put("node.master", false, "node.data", true).build());
final String masterNode = masterFuture.get();
final String node1 = nodesFutures.get().get(0);
final String node2 = nodesFutures.get().get(1);
final String node3 = nodesFutures.get().get(2);
// we will use this later on, handy to start now to make sure it has a different data folder that node 1,2 &3
final String node4 = nodesFutures.get().get(3);
assertAcked(prepareCreate("test").setSettings(Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
.put(FilterAllocationDecider.INDEX_ROUTING_EXCLUDE_GROUP + "_name", node4)
));
assertFalse(client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).setWaitForGreenStatus().setWaitForNodes("5").get().isTimedOut());
// disable allocation to control the situation more easily
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
.put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none")));
logger.debug("--> shutting down two random nodes");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1, node2, node3));
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1, node2, node3));
logger.debug("--> verifying index is red");
ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
if (health.getStatus() != ClusterHealthStatus.RED) {
logClusterState();
fail("cluster didn't become red, despite of shutting 2 of 3 nodes");
}
logger.debug("--> allowing index to be assigned to node [{}]", node4);
assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(
Settings.builder()
.put(FilterAllocationDecider.INDEX_ROUTING_EXCLUDE_GROUP + "_name", "NONE")));
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
.put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "all")));
logger.debug("--> waiting for shards to recover on [{}]", node4);
// we have to do this in two steps as we now do async shard fetching before assigning, so the change to the
// allocation filtering may not have immediate effect
// TODO: we should add an easier to do this. It's too much of a song and dance..
assertBusy(new Runnable() {
@Override
public void run() {
assertTrue(internalCluster().getInstance(IndicesService.class, node4).hasIndex("test"));
}
});
// wait for 4 active shards - we should have lost one shard
assertFalse(client().admin().cluster().prepareHealth().setWaitForActiveShards(4).get().isTimedOut());
// disable allocation again to control concurrency a bit and allow shard active to kick in before allocation
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
.put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none")));
logger.debug("--> starting the two old nodes back");
internalCluster().startNodesAsync(2,
Settings.builder().put("node.master", false, "node.data", true).build());
assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("5").get().isTimedOut());
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
.put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "all")));
logger.debug("--> waiting for the lost shard to be recovered");
ensureGreen("test");
}
@Test
@Slow
public void testShardActiveElseWhere() throws Exception {

View File

@ -356,7 +356,7 @@ public class SimpleIndicesWarmerTests extends ElasticsearchIntegrationTest {
logger.info("register warmer with query cache, validate caching happened");
assertAcked(client().admin().indices().preparePutWarmer("warmer_1")
.setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery()).setQueryCache(true))
.setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery()).setRequestCache(true))
.get());
// index again, to make sure it gets refreshed

View File

@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.metrics;
import com.google.common.collect.Lists;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.script.Script;
@ -50,6 +51,7 @@ import static org.hamcrest.Matchers.sameInstance;
/**
*
*/
@AwaitsFix(bugUrl = "single test methods fail with occassional seeds (see HDRPercentilesTests.testScript_ExplicitSingleValued_WithParams for example) but only if run as a whole test class not if run as a single test method")
public class HDRPercentileRanksTests extends AbstractNumericTests {
private static double[] randomPercents(long minValue, long maxValue) {
@ -256,6 +258,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTests {
@Override
@Test
@AwaitsFix(bugUrl="Fails with seed: B75FCDC119D90BBE, Colin to fix")
public void testSingleValuedField_WithValueScript_WithParams() throws Exception {
int sigDigits = randomSignificantDigits();
Map<String, Object> params = new HashMap<>();
@ -369,6 +372,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTests {
@Override
@Test
@AwaitsFix(bugUrl="Fails with seed: B75FCDC119D90BBE, Colin to fix")
public void testScript_SingleValued_WithParams() throws Exception {
int sigDigits = randomSignificantDigits();
Map<String, Object> params = new HashMap<>();
@ -499,4 +503,4 @@ public class HDRPercentileRanksTests extends AbstractNumericTests {
}
}
}
}

View File

@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.metrics;
import com.google.common.collect.Lists;
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.script.Script;
@ -51,6 +52,7 @@ import static org.hamcrest.Matchers.sameInstance;
/**
*
*/
@AwaitsFix(bugUrl = "single test methods fail with occassional seeds (see testScript_ExplicitSingleValued_WithParams for example) but only if run as a whole test class not if run as a single test method")
public class HDRPercentilesTests extends AbstractNumericTests {
private static double[] randomPercentiles() {
@ -377,6 +379,7 @@ public class HDRPercentilesTests extends AbstractNumericTests {
@Override
@Test
@AwaitsFix(bugUrl = "fails with -Dtests.seed=5BFFA768633A0A59 but only if run as a whole test class not if run as a single test method")
public void testScript_ExplicitSingleValued_WithParams() throws Exception {
Map<String, Object> params = new HashMap<>();
params.put("dec", 1);

View File

@ -60,4 +60,6 @@ com.google.common.collect.Iterators#emptyIterator() @ Use Collections.emptyItera
java.io.ObjectOutputStream
java.io.ObjectOutput
java.io.ObjectInputStream
java.io.ObjectInput
java.io.ObjectInput
java.nio.file.Files#isHidden(java.nio.file.Path) @ Dependent on the operating system, use FileSystemUtils.isHidden instead

280
distribution/deb/pom.xml Normal file
View File

@ -0,0 +1,280 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.elasticsearch.distribution</groupId>
<artifactId>elasticsearch-distribution</artifactId>
<version>2.0.0-beta1-SNAPSHOT</version>
</parent>
<artifactId>elasticsearch-deb</artifactId>
<packaging>deb</packaging>
<name>Elasticsearch DEB Distribution</name>
<properties>
<skip.integ.tests>true</skip.integ.tests>
<deb.sign>false</deb.sign>
<deb.sign.method>dpkg-sig</deb.sign.method>
</properties>
<build>
<filters>
<filter>${project.basedir}/src/main/packaging/packaging.properties</filter>
</filters>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<!-- Filters some files and uses packaging.properties when building the .deb package -->
<execution>
<id>copy-resources-deb</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/generated-packaging/deb/</outputDirectory>
<filters>
<filter>${project.basedir}/../src/main/packaging/packaging.properties</filter>
<filter>${project.basedir}/src/main/packaging/packaging.properties</filter>
</filters>
<resources>
<resource>
<directory>${project.basedir}/../src/main/packaging/</directory>
<filtering>true</filtering>
<includes>
<include>**/*</include>
</includes>
<excludes>
<exclude>packaging.properties</exclude>
</excludes>
</resource>
<resource>
<directory>${project.basedir}/src/main/packaging/</directory>
<filtering>true</filtering>
<includes>
<include>**/*</include>
</includes>
<excludes>
<exclude>packaging.properties</exclude>
</excludes>
</resource>
<resource>
<directory>${project.basedir}/../src/main/resources</directory>
<filtering>true</filtering>
<includes>
<include>bin/elasticsearch</include>
<include>bin/elasticsearch.in.sh</include>
<include>bin/plugin</include>
</includes>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<!-- some infos https://github.com/tcurdt/jdeb/blob/master/docs/maven.md -->
<artifactId>jdeb</artifactId>
<groupId>org.vafer</groupId>
<version>1.4</version>
<configuration>
<!-- By default it should generates target/${artifactId}_${version}.deb but we get elasticsearch_2.0.0~SNAPSHOT_all.deb -->
<deb>${project.build.directory}/releases/elasticsearch-${project.version}.deb</deb>
<controlDir>${project.build.directory}/generated-packaging/deb/scripts</controlDir>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>jdeb</goal>
</goals>
<configuration>
<signPackage>${deb.sign}</signPackage>
<keyring>${gpg.keyring}</keyring>
<key>${gpg.key}</key>
<passphrase>${gpg.passphrase}</passphrase>
<signMethod>${deb.sign.method}</signMethod>
<dataSet>
<!-- Add bin directory -->
<data>
<src>${project.build.directory}/generated-packaging/deb/bin</src>
<type>directory</type>
<includes>elasticsearch,elasticsearch.in.sh,plugin</includes>
<mapper>
<type>perm</type>
<prefix>${packaging.elasticsearch.bin.dir}</prefix>
<filemode>755</filemode>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add configuration files -->
<data>
<src>${project.basedir}/../src/main/resources/config</src>
<type>directory</type>
<excludes>.DS_Store</excludes>
<mapper>
<type>perm</type>
<prefix>${packaging.elasticsearch.conf.dir}</prefix>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add environment vars file -->
<data>
<src>${project.build.directory}/generated-packaging/deb/env/elasticsearch</src>
<type>file</type>
<mapper>
<type>perm</type>
<prefix>/etc/default</prefix>
<filemode>644</filemode>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add libs -->
<data>
<src>${project.build.directory}/</src>
<includes>elasticsearch-${project.version}.jar</includes>
<type>directory</type>
<mapper>
<type>perm</type>
<prefix>${packaging.elasticsearch.home.dir}/lib</prefix>
<user>root</user>
<group>root</group>
</mapper>
</data>
<data>
<src>${project.build.directory}/../target/lib</src>
<excludes>${project.build.finalName}-shaded.jar,${project.build.finalName}-sources.jar,${project.build.finalName}-tests.jar,${project.build.finalName}-test-sources.jar,slf4j-api-*.jar</excludes>
<type>directory</type>
<mapper>
<type>perm</type>
<prefix>${packaging.elasticsearch.home.dir}/lib</prefix>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add init.d files -->
<data>
<src>${project.build.directory}/generated-packaging/deb/init.d/</src>
<type>directory</type>
<excludes>.DS_Store</excludes>
<mapper>
<type>perm</type>
<prefix>/etc/init.d</prefix>
<filemode>755</filemode>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Adds systemd file -->
<data>
<src>${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.service</src>
<dst>${packaging.elasticsearch.systemd.dir}/elasticsearch.service</dst>
<type>file</type>
</data>
<!-- Adds systemd/sysctl.d configuration file -->
<data>
<src>${project.build.directory}/generated-packaging/deb/systemd/sysctl/elasticsearch.conf</src>
<dst>${packaging.elasticsearch.systemd.sysctl.dir}/elasticsearch.conf</dst>
<type>file</type>
</data>
<!-- Adds systemd/tmpfiles.d configuration file -->
<data>
<src>${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.conf</src>
<dst>${packaging.elasticsearch.tmpfilesd.dir}/elasticsearch.conf</dst>
<type>file</type>
</data>
<!-- Add lintian files -->
<data>
<src>${project.build.directory}/generated-packaging/deb/lintian</src>
<type>directory</type>
<excludes>.DS_Store</excludes>
<mapper>
<type>perm</type>
<prefix>/usr/share/lintian/overrides</prefix>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add readme files -->
<data>
<src>${project.basedir}/../src/main/resources/</src>
<includes>*.txt, *.textile</includes>
<excludes>LICENSE.txt, .DS_Store</excludes>
<type>directory</type>
<mapper>
<type>perm</type>
<prefix>${packaging.elasticsearch.home.dir}</prefix>
<user>root</user>
<group>root</group>
</mapper>
</data>
<!-- Add license files -->
<data>
<src>${project.build.directory}/generated-packaging/deb/copyright</src>
<dst>/usr/share/doc/elasticsearch/copyright</dst>
<type>file</type>
</data>
<!-- Adds and sets permission on default directories -->
<data>
<type>template</type>
<paths>
<path>${packaging.elasticsearch.data.dir}</path>
<path>${packaging.elasticsearch.log.dir}</path>
<path>${packaging.elasticsearch.plugins.dir}</path>
<path>${packaging.elasticsearch.pid.dir}</path>
</paths>
<mapper>
<type>perm</type>
<dirmode>755</dirmode>
<user>${packaging.elasticsearch.user}</user>
<group>${packaging.elasticsearch.group}</group>
</mapper>
</data>
</dataSet>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<id>check-license</id>
<phase>verify</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<condition property="licenses.exists">
<available file="${basedir}/../licenses" type="dir"/>
</condition>
<echo taskName="license check">Running license check</echo>
<!-- on windows checksums are calculated wrong -->
<exec failonerror="${licenses.exists}" executable="perl" dir="${elasticsearch.tools.directory}/license-check" osfamily="unix" >
<arg value="check_license_and_sha.pl"/>
<arg value="--check"/>
<arg value="${basedir}/../licenses"/>
<arg value="${basedir}/target/releases/${project.build.finalName}.deb"/>
</exec>
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@ -173,7 +173,7 @@ case "$1" in
fi
# Start Daemon
start-stop-daemon --start -b --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
start-stop-daemon -d $ES_HOME --start -b --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
return=$?
if [ $return -eq 0 ]
then

Some files were not shown because too many files have changed in this diff Show More