diff --git a/TESTING.asciidoc b/TESTING.asciidoc index e471bf858f2..212e215a201 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -44,7 +44,7 @@ In order to run Elasticsearch from source without building a package, you can run it using Maven: ------------------------------------- -mvn compile exec:exec +mvn package -Drun -DskipTests ------------------------------------- === Test case filtering. @@ -318,4 +318,6 @@ environment before the tests are executed: ES_CLEAN_BEFORE_TEST=true bats 30_deb_package.bats --------------------------------------------------------------------------- - +The current mode of execution is to copy all the packages that should be tested +into one directory, then copy the bats files into the same directory and run +those. diff --git a/core/lib/sigar/libsigar-amd64-freebsd-6.so b/core/lib/sigar/libsigar-amd64-freebsd-6.so deleted file mode 100644 index 3e94f0d2bfb..00000000000 Binary files a/core/lib/sigar/libsigar-amd64-freebsd-6.so and /dev/null differ diff --git a/core/lib/sigar/libsigar-amd64-linux.so b/core/lib/sigar/libsigar-amd64-linux.so deleted file mode 100644 index 5a2e4c24feb..00000000000 Binary files a/core/lib/sigar/libsigar-amd64-linux.so and /dev/null differ diff --git a/core/lib/sigar/libsigar-amd64-solaris.so b/core/lib/sigar/libsigar-amd64-solaris.so deleted file mode 100644 index 6396482a431..00000000000 Binary files a/core/lib/sigar/libsigar-amd64-solaris.so and /dev/null differ diff --git a/core/lib/sigar/libsigar-ia64-linux.so b/core/lib/sigar/libsigar-ia64-linux.so deleted file mode 100644 index 2bd2fc8e323..00000000000 Binary files a/core/lib/sigar/libsigar-ia64-linux.so and /dev/null differ diff --git a/core/lib/sigar/libsigar-sparc-solaris.so b/core/lib/sigar/libsigar-sparc-solaris.so deleted file mode 100644 index aa847d2b54d..00000000000 Binary files a/core/lib/sigar/libsigar-sparc-solaris.so and /dev/null differ diff --git a/core/lib/sigar/libsigar-sparc64-solaris.so b/core/lib/sigar/libsigar-sparc64-solaris.so deleted file mode 100644 index 6c4fe809c53..00000000000 Binary files a/core/lib/sigar/libsigar-sparc64-solaris.so and /dev/null differ diff --git a/core/lib/sigar/libsigar-universal-macosx.dylib b/core/lib/sigar/libsigar-universal-macosx.dylib deleted file mode 100644 index 27ab1071113..00000000000 Binary files a/core/lib/sigar/libsigar-universal-macosx.dylib and /dev/null differ diff --git a/core/lib/sigar/libsigar-universal64-macosx.dylib b/core/lib/sigar/libsigar-universal64-macosx.dylib deleted file mode 100644 index 0c721fecf3d..00000000000 Binary files a/core/lib/sigar/libsigar-universal64-macosx.dylib and /dev/null differ diff --git a/core/lib/sigar/libsigar-x86-freebsd-5.so b/core/lib/sigar/libsigar-x86-freebsd-5.so deleted file mode 100644 index 8c50c6117ae..00000000000 Binary files a/core/lib/sigar/libsigar-x86-freebsd-5.so and /dev/null differ diff --git a/core/lib/sigar/libsigar-x86-freebsd-6.so b/core/lib/sigar/libsigar-x86-freebsd-6.so deleted file mode 100644 index f0800274a6a..00000000000 Binary files a/core/lib/sigar/libsigar-x86-freebsd-6.so and /dev/null differ diff --git a/core/lib/sigar/libsigar-x86-linux.so b/core/lib/sigar/libsigar-x86-linux.so deleted file mode 100644 index a0b64eddb0d..00000000000 Binary files a/core/lib/sigar/libsigar-x86-linux.so and /dev/null differ diff --git a/core/lib/sigar/libsigar-x86-solaris.so b/core/lib/sigar/libsigar-x86-solaris.so deleted file mode 100644 index c6452e56559..00000000000 Binary files a/core/lib/sigar/libsigar-x86-solaris.so and /dev/null differ diff --git a/core/lib/sigar/sigar-1.6.4.jar b/core/lib/sigar/sigar-1.6.4.jar deleted file mode 100644 index 58c733c6ad8..00000000000 Binary files a/core/lib/sigar/sigar-1.6.4.jar and /dev/null differ diff --git a/core/lib/sigar/sigar-amd64-winnt.dll b/core/lib/sigar/sigar-amd64-winnt.dll deleted file mode 100644 index 1ec8a0353e6..00000000000 Binary files a/core/lib/sigar/sigar-amd64-winnt.dll and /dev/null differ diff --git a/core/lib/sigar/sigar-x86-winnt.dll b/core/lib/sigar/sigar-x86-winnt.dll deleted file mode 100644 index 6afdc0166c1..00000000000 Binary files a/core/lib/sigar/sigar-x86-winnt.dll and /dev/null differ diff --git a/core/lib/sigar/sigar-x86-winnt.lib b/core/lib/sigar/sigar-x86-winnt.lib deleted file mode 100644 index 04924a1fc1d..00000000000 Binary files a/core/lib/sigar/sigar-x86-winnt.lib and /dev/null differ diff --git a/core/pom.xml b/core/pom.xml index b7383d3783a..b507a36707d 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -18,24 +18,9 @@ - - /usr/share/elasticsearch - /usr/share/elasticsearch/bin - /etc/elasticsearch - /var/lib/elasticsearch - elasticsearch - elasticsearch - /var/log/elasticsearch - ${packaging.elasticsearch.home.dir}/plugins - /var/run/elasticsearch - /usr/lib/systemd/system - /usr/lib/sysctl.d - /usr/lib/tmpfiles.d - false - dpkg-sig + true - org.hamcrest @@ -226,27 +211,10 @@ 1.1.0 - - - - - - ${project.basedir}/src/packaging/common/packaging.properties - - ${project.basedir}/src/main/java @@ -442,532 +410,6 @@ - - maven-resources-plugin - 2.6 - - - copy-resources - prepare-package - - copy-resources - - - ${project.build.directory}/bin - - - ${project.basedir}/bin - true - - *.exe - - - - ${project.basedir}/bin - false - - *.exe - - - - - - - - copy-resources-deb - prepare-package - - copy-resources - - - ${project.build.directory}/generated-packaging/deb/ - - ${project.basedir}/src/packaging/common/packaging.properties - ${project.basedir}/src/packaging/deb/packaging.properties - - - - ${project.basedir}/src/packaging/common/ - true - - **/* - - - packaging.properties - - - - ${project.basedir}/src/packaging/deb/ - true - - **/* - - - packaging.properties - - - - ${project.basedir} - true - - bin/elasticsearch - bin/elasticsearch.in.sh - bin/plugin - - - - - - - - copy-resources-rpm - prepare-package - - copy-resources - - - ${project.build.directory}/generated-packaging/rpm/ - - ${project.basedir}/src/packaging/common/packaging.properties - ${project.basedir}/src/packaging/rpm/packaging.properties - - - - src/packaging/common/ - true - - **/* - - - packaging.properties - - - - src/packaging/rpm/ - true - - **/* - - - packaging.properties - - - - ${project.basedir} - true - - bin/elasticsearch - bin/elasticsearch.in.sh - bin/plugin - - - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - false - ${project.build.directory}/releases/ - - ${project.basedir}/src/main/assemblies/targz-bin.xml - ${project.basedir}/src/main/assemblies/zip-bin.xml - - - - - package - - single - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - prepare-package - - copy-dependencies - - - ${project.build.directory}/lib - runtime - - - - - - - - - jdeb - org.vafer - 1.4 - - ${project.build.directory}/releases/${project.artifactId}-${project.version}.deb - ${project.build.directory}/generated-packaging/deb/scripts - - - - package - - jdeb - - - ${deb.sign} - ${gpg.keyring} - ${gpg.key} - ${gpg.passphrase} - ${deb.sign.method} - - - - ${project.build.directory}/generated-packaging/deb/bin - directory - elasticsearch,elasticsearch.in.sh,plugin - - perm - ${packaging.elasticsearch.bin.dir} - 755 - root - root - - - - - ${project.basedir}/config - directory - .DS_Store - - perm - ${packaging.elasticsearch.conf.dir} - root - root - - - - - ${project.build.directory}/generated-packaging/deb/env/elasticsearch - file - - perm - /etc/default - 644 - root - root - - - - - ${project.build.directory}/ - ${project.build.finalName}.jar - directory - - perm - ${packaging.elasticsearch.home.dir}/lib - root - root - - - - ${project.build.directory}/lib - ${project.build.finalName}-shaded.jar,${project.build.finalName}-sources.jar,${project.build.finalName}-tests.jar,${project.build.finalName}-test-sources.jar,slf4j-api-*.jar - directory - - perm - ${packaging.elasticsearch.home.dir}/lib - root - root - - - - - ${project.build.directory}/generated-packaging/deb/init.d/ - directory - .DS_Store - - perm - /etc/init.d - 755 - root - root - - - - - ${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.service - ${packaging.elasticsearch.systemd.dir}/elasticsearch.service - file - - - - ${project.build.directory}/generated-packaging/deb/systemd/sysctl/elasticsearch.conf - ${packaging.elasticsearch.systemd.sysctl.dir}/elasticsearch.conf - file - - - - ${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.conf - ${packaging.elasticsearch.tmpfilesd.dir}/elasticsearch.conf - file - - - - ${project.build.directory}/generated-packaging/deb/lintian - directory - .DS_Store - - perm - /usr/share/lintian/overrides - root - root - - - - - ${project.basedir}/ - *.txt, *.textile - LICENSE.txt, .DS_Store - directory - - perm - ${packaging.elasticsearch.home.dir} - root - root - - - - - ${project.build.directory}/generated-packaging/deb/copyright - /usr/share/doc/elasticsearch/copyright - file - - - - template - - ${packaging.elasticsearch.data.dir} - ${packaging.elasticsearch.log.dir} - ${packaging.elasticsearch.plugins.dir} - ${packaging.elasticsearch.pid.dir} - - - perm - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - - - - - - - - - - org.codehaus.mojo - rpm-maven-plugin - 2.1.3 - - Elasticsearch - Application/Internet - Elasticsearch - /usr - src/changelog - - _unpackaged_files_terminate_build 0 - _binaries_in_noarch_packages_terminate_build 0 - - 644 - 755 - root - root - - - - ${packaging.elasticsearch.bin.dir}/ - 755 - - - ${project.build.directory}/generated-packaging/rpm/bin - - elasticsearch - elasticsearch.in.sh - plugin - - - - - - - ${packaging.elasticsearch.conf.dir}/ - noreplace - - - ${project.basedir}/config/ - - *.yml - - - - - - - /etc/sysconfig/ - false - noreplace - - - ${project.build.directory}/generated-packaging/rpm/env/ - - elasticsearch - - - - - - - ${packaging.elasticsearch.home.dir}/lib - - - target/lib/ - - ${project.build.finalName}-shaded.jar - ${project.build.finalName}-sources.jar - ${project.build.finalName}-tests.jar - ${project.build.finalName}-test-sources.jar - slf4j-api-*.jar - - - - ${project.build.directory}/ - - ${project.build.finalName}.jar - - - - - - - /etc/init.d - false - 755 - true - - - ${project.build.directory}/generated-packaging/rpm/init.d - - elasticsearch - - - - - - - ${packaging.elasticsearch.systemd.dir} - false - true - - - ${project.build.directory}/generated-packaging/rpm/systemd - - elasticsearch.service - - - - - - - ${packaging.elasticsearch.systemd.sysctl.dir} - true - - - ${project.build.directory}/generated-packaging/rpm/systemd/sysctl - - elasticsearch.conf - - - - - - - ${packaging.elasticsearch.tmpfilesd.dir} - true - - - ${project.build.directory}/generated-packaging/rpm/systemd/ - - elasticsearch.conf - - - - - - - ${packaging.elasticsearch.home.dir} - - - . - - LICENSE.txt - NOTICE.txt - README.textile - - - - - - - ${packaging.elasticsearch.data.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - ${packaging.elasticsearch.log.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - ${packaging.elasticsearch.plugins.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - ${packaging.elasticsearch.pid.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - - ${project.build.directory}/generated-packaging/rpm/scripts/preinst - utf-8 - - - ${project.build.directory}/generated-packaging/rpm/scripts/postinst - utf-8 - - - ${project.build.directory}/generated-packaging/rpm/scripts/prerm - utf-8 - - - ${project.build.directory}/generated-packaging/rpm/scripts/postrm - utf-8 - - - org.apache.maven.plugins @@ -1014,32 +456,6 @@ - - - integ-setup - pre-integration-test - - run - - - - - - - - - - integ-teardown - post-integration-test - - run - - - - - - - @@ -1085,54 +501,5 @@ - - release - - - package.rpm - true - - - - - - org.codehaus.mojo - rpm-maven-plugin - - - attach-rpm - - attached-rpm - - - - - - - - - sign-rpm - - - rpm.sign - true - - - - - - org.codehaus.mojo - rpm-maven-plugin - - ${gpg.key} - ${gpg.keyring} - - ${gpg.passphrase} - - - - - - diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java index 046fb04028e..0e77e3d6e3a 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -395,7 +395,6 @@ public class ClusterStatsNodes implements ToXContent, Streamable { } count++; if (nodeStats.getProcess().getCpu() != null) { - // with no sigar, this may not be available cpuPercent += nodeStats.getProcess().getCpu().getPercent(); } long fd = nodeStats.getProcess().getOpenFileDescriptors(); diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java index 54f8c861bea..06e7d03b98d 100644 --- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java @@ -130,8 +130,8 @@ public class MultiSearchRequest extends ActionRequest implem searchRequest.types(nodeStringArrayValue(value)); } else if ("search_type".equals(entry.getKey()) || "searchType".equals(entry.getKey())) { searchRequest.searchType(nodeStringValue(value, null)); - } else if ("query_cache".equals(entry.getKey()) || "queryCache".equals(entry.getKey())) { - searchRequest.queryCache(nodeBooleanValue(value)); + } else if ("request_cache".equals(entry.getKey()) || "requestCache".equals(entry.getKey())) { + searchRequest.requestCache(nodeBooleanValue(value)); } else if ("preference".equals(entry.getKey())) { searchRequest.preference(nodeStringValue(value, null)); } else if ("routing".equals(entry.getKey())) { diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 7d67e970c7c..c423188fc66 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -529,12 +529,12 @@ public class SearchRequest extends ActionRequest implements Indic } /** - * Sets if this request should use the query cache or not, assuming that it can (for + * Sets if this request should use the request cache or not, assuming that it can (for * example, if "now" is used, it will never be cached). By default (not set, or null, - * will default to the index level setting if query cache is enabled or not). + * will default to the index level setting if request cache is enabled or not). */ - public SearchRequest queryCache(Boolean queryCache) { - this.requestCache = queryCache; + public SearchRequest requestCache(Boolean requestCache) { + this.requestCache = requestCache; return this; } diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index 1139670d527..f6365342a69 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -957,12 +957,12 @@ public class SearchRequestBuilder extends ActionRequestBuilder stream = Files.newDirectoryStream(mappingsDir)) { for (Path mappingFile : stream) { final String fileName = mappingFile.getFileName().toString(); - if (Files.isHidden(mappingFile)) { + if (FileSystemUtils.isHidden(mappingFile)) { continue; } int lastDotIndex = fileName.lastIndexOf('.'); diff --git a/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java b/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java index 50f845f754a..bf7eb5ee28d 100644 --- a/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java +++ b/core/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java @@ -20,6 +20,7 @@ package org.elasticsearch.common.io; import com.google.common.collect.Iterators; + import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.logging.ESLogger; @@ -84,6 +85,20 @@ public final class FileSystemUtils { return false; } + /** + * Check whether the file denoted by the given path is hidden. + * In practice, this will check if the file name starts with a dot. + * This should be preferred to {@link Files#isHidden(Path)} as this + * does not depend on the operating system. + */ + public static boolean isHidden(Path path) { + Path fileName = path.getFileName(); + if (fileName == null) { + return false; + } + return fileName.toString().startsWith("."); + } + /** * Appends the path to the given base and strips N elements off the path if strip is > 0. */ @@ -334,4 +349,5 @@ public final class FileSystemUtils { return Iterators.toArray(stream.iterator(), Path.class); } } + } diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 8f0762b85b4..f01438a3a0f 100644 --- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -22,6 +22,7 @@ package org.elasticsearch.env; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import com.google.common.primitives.Ints; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.*; @@ -29,6 +30,7 @@ import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; @@ -661,6 +663,56 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { return indices; } + /** + * Tries to find all allocated shards for the given index + * on the current node. NOTE: This methods is prone to race-conditions on the filesystem layer since it might not + * see directories created concurrently or while it's traversing. + * @param index the index to filter shards + * @return a set of shard IDs + * @throws IOException if an IOException occurs + */ + public Set findAllShardIds(final Index index) throws IOException { + assert index != null; + if (nodePaths == null || locks == null) { + throw new IllegalStateException("node is not configured to store local location"); + } + assert assertEnvIsLocked(); + final Set shardIds = Sets.newHashSet(); + String indexName = index.name(); + for (final NodePath nodePath : nodePaths) { + Path location = nodePath.indicesPath; + if (Files.isDirectory(location)) { + try (DirectoryStream indexStream = Files.newDirectoryStream(location)) { + for (Path indexPath : indexStream) { + if (indexName.equals(indexPath.getFileName().toString())) { + shardIds.addAll(findAllShardsForIndex(indexPath)); + } + } + } + } + } + return shardIds; + } + + private static Set findAllShardsForIndex(Path indexPath) throws IOException { + Set shardIds = new HashSet<>(); + if (Files.isDirectory(indexPath)) { + try (DirectoryStream stream = Files.newDirectoryStream(indexPath)) { + String currentIndex = indexPath.getFileName().toString(); + for (Path shardPath : stream) { + if (Files.isDirectory(shardPath)) { + Integer shardId = Ints.tryParse(shardPath.getFileName().toString()); + if (shardId != null) { + ShardId id = new ShardId(currentIndex, shardId); + shardIds.add(id); + } + } + } + } + } + return shardIds; + } + @Override public void close() { if (closed.compareAndSet(false, true) && locks != null) { diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java index 5199ea5e422..40067e2bff3 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpServer.java +++ b/core/src/main/java/org/elasticsearch/http/HttpServer.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.node.service.NodeService; @@ -183,7 +184,7 @@ public class HttpServer extends AbstractLifecycleComponent { Path file = siteFile.resolve(sitePath); // return not found instead of forbidden to prevent malicious requests to find out if files exist or dont exist - if (!Files.exists(file) || Files.isHidden(file) || !file.toAbsolutePath().normalize().startsWith(siteFile.toAbsolutePath().normalize())) { + if (!Files.exists(file) || FileSystemUtils.isHidden(file) || !file.toAbsolutePath().normalize().startsWith(siteFile.toAbsolutePath().normalize())) { channel.sendResponse(new BytesRestResponse(NOT_FOUND)); return; } @@ -197,7 +198,7 @@ public class HttpServer extends AbstractLifecycleComponent { } // We don't serve dir but if index.html exists in dir we should serve it file = file.resolve("index.html"); - if (!Files.exists(file) || Files.isHidden(file) || !Files.isRegularFile(file)) { + if (!Files.exists(file) || FileSystemUtils.isHidden(file) || !Files.isRegularFile(file)) { channel.sendResponse(new BytesRestResponse(FORBIDDEN)); return; } diff --git a/core/src/main/java/org/elasticsearch/index/cache/request/RequestCacheStats.java b/core/src/main/java/org/elasticsearch/index/cache/request/RequestCacheStats.java index 58bb6a69379..9186fadbcee 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/request/RequestCacheStats.java +++ b/core/src/main/java/org/elasticsearch/index/cache/request/RequestCacheStats.java @@ -93,7 +93,7 @@ public class RequestCacheStats implements Streamable, ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(Fields.QUERY_CACHE_STATS); + builder.startObject(Fields.REQUEST_CACHE_STATS); builder.byteSizeField(Fields.MEMORY_SIZE_IN_BYTES, Fields.MEMORY_SIZE, memorySize); builder.field(Fields.EVICTIONS, getEvictions()); builder.field(Fields.HIT_COUNT, getHitCount()); @@ -103,7 +103,7 @@ public class RequestCacheStats implements Streamable, ToXContent { } static final class Fields { - static final XContentBuilderString QUERY_CACHE_STATS = new XContentBuilderString("request_cache"); + static final XContentBuilderString REQUEST_CACHE_STATS = new XContentBuilderString("request_cache"); static final XContentBuilderString MEMORY_SIZE = new XContentBuilderString("memory_size"); static final XContentBuilderString MEMORY_SIZE_IN_BYTES = new XContentBuilderString("memory_size_in_bytes"); static final XContentBuilderString EVICTIONS = new XContentBuilderString("evictions"); diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java index 912a1ded50e..d207e4b6618 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -524,18 +524,38 @@ public class IndicesService extends AbstractLifecycleComponent i * This method deletes the shard contents on disk for the given shard ID. This method will fail if the shard deleting * is prevented by {@link #canDeleteShardContent(org.elasticsearch.index.shard.ShardId, org.elasticsearch.cluster.metadata.IndexMetaData)} * of if the shards lock can not be acquired. + * + * On data nodes, if the deleted shard is the last shard folder in its index, the method will attempt to remove the index folder as well. + * * @param reason the reason for the shard deletion * @param shardId the shards ID to delete - * @param metaData the shards index metadata. This is required to access the indexes settings etc. + * @param clusterState . This is required to access the indexes settings etc. * @throws IOException if an IOException occurs */ - public void deleteShardStore(String reason, ShardId shardId, IndexMetaData metaData) throws IOException { + public void deleteShardStore(String reason, ShardId shardId, ClusterState clusterState) throws IOException { + final IndexMetaData metaData = clusterState.getMetaData().indices().get(shardId.getIndex()); + final Settings indexSettings = buildIndexSettings(metaData); if (canDeleteShardContent(shardId, indexSettings) == false) { throw new IllegalStateException("Can't delete shard " + shardId); } nodeEnv.deleteShardDirectorySafe(shardId, indexSettings); - logger.trace("{} deleting shard reason [{}]", shardId, reason); + logger.debug("{} deleted shard reason [{}]", shardId, reason); + + if (clusterState.nodes().localNode().isMasterNode() == false && // master nodes keep the index meta data, even if having no shards.. + canDeleteIndexContents(shardId.index(), indexSettings)) { + if (nodeEnv.findAllShardIds(shardId.index()).isEmpty()) { + try { + // note that deleteIndexStore have more safety checks and may throw an exception if index was concurrently created. + deleteIndexStore("no longer used", metaData, clusterState); + } catch (Exception e) { + // wrap the exception to indicate we already deleted the shard + throw new ElasticsearchException("failed to delete unused index after deleting its last shard (" + shardId + ")", e); + } + } else { + logger.trace("[{}] still has shard stores, leaving as is", shardId.index()); + } + } } /** diff --git a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index 493d13a854d..f87e2c4e911 100644 --- a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -21,7 +21,6 @@ package org.elasticsearch.indices.store; import org.apache.lucene.store.StoreRateLimiting; import org.elasticsearch.cluster.*; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -288,28 +287,18 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe return; } - clusterService.submitStateUpdateTask("indices_store", new ClusterStateNonMasterUpdateTask() { + clusterService.submitStateUpdateTask("indices_store ([" + shardId + "] active fully on other nodes)", new ClusterStateNonMasterUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { if (clusterState.getVersion() != currentState.getVersion()) { logger.trace("not deleting shard {}, the update task state version[{}] is not equal to cluster state before shard active api call [{}]", shardId, currentState.getVersion(), clusterState.getVersion()); return currentState; } - IndexMetaData indexMeta = clusterState.getMetaData().indices().get(shardId.getIndex()); try { - indicesService.deleteShardStore("no longer used", shardId, indexMeta); + indicesService.deleteShardStore("no longer used", shardId, currentState); } catch (Throwable ex) { logger.debug("{} failed to delete unallocated shard, ignoring", ex, shardId); } - // if the index doesn't exists anymore, delete its store as well, but only if its a non master node, since master - // nodes keep the index metadata around - if (indicesService.hasIndex(shardId.getIndex()) == false && currentState.nodes().localNode().masterNode() == false) { - try { - indicesService.deleteIndexStore("no longer used", indexMeta, currentState); - } catch (Throwable ex) { - logger.debug("{} failed to delete unallocated index, ignoring", ex, shardId.getIndex()); - } - } return currentState; } diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java index abb313c0a82..70494d71f82 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo; import org.elasticsearch.bootstrap.Bootstrap; import org.elasticsearch.bootstrap.JarHell; @@ -33,6 +32,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; @@ -40,7 +40,6 @@ import org.elasticsearch.env.Environment; import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; @@ -54,7 +53,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Set; import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory; @@ -312,7 +310,7 @@ public class PluginsService extends AbstractComponent { try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory)) { for (Path plugin : stream) { try { - if (Files.isHidden(plugin)) { + if (FileSystemUtils.isHidden(plugin)) { logger.trace("--- skip hidden plugin file[{}]", plugin.toAbsolutePath()); continue; } diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 4af5b7c00f4..62c57ed21d6 100644 --- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -561,7 +561,9 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent source = new BytesArray(parser.binaryValue()); } } else if (token.isValue()) { - if ("queryCache".equals(currentFieldName) || "query_cache".equals(currentFieldName)) { + if ("requestCache".equals(currentFieldName) || "request_cache".equals(currentFieldName)) { queryCache = parser.booleanValue(); } } @@ -242,7 +242,7 @@ public class IndexWarmersMetaData extends AbstractDiffable builder.startObject(entry.name(), XContentBuilder.FieldCaseConversion.NONE); builder.field("types", entry.types()); if (entry.requestCache() != null) { - builder.field("queryCache", entry.requestCache()); + builder.field("requestCache", entry.requestCache()); } builder.field("source"); if (binary) { diff --git a/core/src/test/java/org/elasticsearch/action/search/simple-msearch4.json b/core/src/test/java/org/elasticsearch/action/search/simple-msearch4.json index ab6b8206b01..b98e24b8660 100644 --- a/core/src/test/java/org/elasticsearch/action/search/simple-msearch4.json +++ b/core/src/test/java/org/elasticsearch/action/search/simple-msearch4.json @@ -1,4 +1,4 @@ -{"index":["test0", "test1"], "query_cache": true} +{"index":["test0", "test1"], "request_cache": true} {"query" : {"match_all" {}}} {"index" : "test2,test3", "type" : "type1", "preference": "_local"} {"query" : {"match_all" {}}} diff --git a/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java b/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java index aa8c56cc97a..53d7c82de30 100644 --- a/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/FileSystemUtilsTests.java @@ -32,6 +32,7 @@ import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists; @@ -173,4 +174,39 @@ public class FileSystemUtilsTests extends ElasticsearchTestCase { assertEquals(FileSystemUtils.append(PathUtils.get("/foo/bar"), PathUtils.get("/hello/world/this_is/awesome"), 1), PathUtils.get("/foo/bar/world/this_is/awesome")); } + + public void testIsHidden() { + for (String p : Arrays.asList( + "/", + "foo", + "/foo", + "foo.bar", + "/foo.bar", + "foo/bar", + "foo/./bar", + "foo/../bar", + "/foo/./bar", + "/foo/../bar" + )) { + Path path = PathUtils.get(p); + assertFalse(FileSystemUtils.isHidden(path)); + } + for (String p : Arrays.asList( + ".hidden", + ".hidden.ext", + "/.hidden", + "/.hidden.ext", + "foo/.hidden", + "foo/.hidden.ext", + "/foo/.hidden", + "/foo/.hidden.ext", + ".", + "..", + "foo/.", + "foo/.." + )) { + Path path = PathUtils.get(p); + assertTrue(FileSystemUtils.isHidden(path)); + } + } } diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java index b32ad01a491..ad446251db9 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java @@ -259,10 +259,10 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest { // test explicit request parameter - assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setQueryCache(false).get().getHits().getTotalHits(), equalTo((long) numDocs)); + assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(false).get().getHits().getTotalHits(), equalTo((long) numDocs)); assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0l)); - assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setQueryCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs)); + assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs)); assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0l)); // set the index level setting to false, and see that the reverse works @@ -273,7 +273,7 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest { assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get().getHits().getTotalHits(), equalTo((long) numDocs)); assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0l)); - assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setQueryCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs)); + assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs)); assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0l)); } diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java index 8162c951534..ac43f0e8a8d 100644 --- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java +++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.indices.store; import com.google.common.base.Predicate; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; @@ -30,6 +31,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; import org.elasticsearch.common.Priority; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; @@ -37,12 +39,14 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.RecoverySource; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.disruption.BlockClusterStateProcessing; import org.elasticsearch.test.disruption.SingleNodeDisruption; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportModule; import org.elasticsearch.transport.TransportRequestOptions; @@ -55,6 +59,7 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import static java.lang.Thread.sleep; @@ -217,6 +222,87 @@ public class IndicesStoreIntegrationTests extends ElasticsearchIntegrationTest { assertThat(waitForShardDeletion(node_4, "test", 0), equalTo(false)); } + + @Test + @TestLogging("cluster.service:TRACE") + public void testShardActiveElsewhereDoesNotDeleteAnother() throws Exception { + Future masterFuture = internalCluster().startNodeAsync( + Settings.builder().put("node.master", true, "node.data", false).build()); + Future> nodesFutures = internalCluster().startNodesAsync(4, + Settings.builder().put("node.master", false, "node.data", true).build()); + + final String masterNode = masterFuture.get(); + final String node1 = nodesFutures.get().get(0); + final String node2 = nodesFutures.get().get(1); + final String node3 = nodesFutures.get().get(2); + // we will use this later on, handy to start now to make sure it has a different data folder that node 1,2 &3 + final String node4 = nodesFutures.get().get(3); + + assertAcked(prepareCreate("test").setSettings(Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(FilterAllocationDecider.INDEX_ROUTING_EXCLUDE_GROUP + "_name", node4) + )); + assertFalse(client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).setWaitForGreenStatus().setWaitForNodes("5").get().isTimedOut()); + + // disable allocation to control the situation more easily + assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder() + .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none"))); + + logger.debug("--> shutting down two random nodes"); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1, node2, node3)); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1, node2, node3)); + + logger.debug("--> verifying index is red"); + ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + if (health.getStatus() != ClusterHealthStatus.RED) { + logClusterState(); + fail("cluster didn't become red, despite of shutting 2 of 3 nodes"); + } + + logger.debug("--> allowing index to be assigned to node [{}]", node4); + assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings( + Settings.builder() + .put(FilterAllocationDecider.INDEX_ROUTING_EXCLUDE_GROUP + "_name", "NONE"))); + + assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder() + .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "all"))); + + logger.debug("--> waiting for shards to recover on [{}]", node4); + // we have to do this in two steps as we now do async shard fetching before assigning, so the change to the + // allocation filtering may not have immediate effect + // TODO: we should add an easier to do this. It's too much of a song and dance.. + assertBusy(new Runnable() { + @Override + public void run() { + assertTrue(internalCluster().getInstance(IndicesService.class, node4).hasIndex("test")); + } + }); + + // wait for 4 active shards - we should have lost one shard + assertFalse(client().admin().cluster().prepareHealth().setWaitForActiveShards(4).get().isTimedOut()); + + // disable allocation again to control concurrency a bit and allow shard active to kick in before allocation + assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder() + .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none"))); + + logger.debug("--> starting the two old nodes back"); + + internalCluster().startNodesAsync(2, + Settings.builder().put("node.master", false, "node.data", true).build()); + + assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("5").get().isTimedOut()); + + + assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder() + .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "all"))); + + logger.debug("--> waiting for the lost shard to be recovered"); + + ensureGreen("test"); + + } + @Test @Slow public void testShardActiveElseWhere() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java b/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java index 6f9a2a1bdc4..f8ac6c156f9 100644 --- a/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java +++ b/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerTests.java @@ -356,7 +356,7 @@ public class SimpleIndicesWarmerTests extends ElasticsearchIntegrationTest { logger.info("register warmer with query cache, validate caching happened"); assertAcked(client().admin().indices().preparePutWarmer("warmer_1") - .setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery()).setQueryCache(true)) + .setSearchRequest(client().prepareSearch("test").setTypes("a1").setQuery(QueryBuilders.matchAllQuery()).setRequestCache(true)) .get()); // index again, to make sure it gets refreshed diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksTests.java index 70767d70a10..989132a6c9c 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.metrics; import com.google.common.collect.Lists; +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.script.Script; @@ -50,6 +51,7 @@ import static org.hamcrest.Matchers.sameInstance; /** * */ +@AwaitsFix(bugUrl = "single test methods fail with occassional seeds (see HDRPercentilesTests.testScript_ExplicitSingleValued_WithParams for example) but only if run as a whole test class not if run as a single test method") public class HDRPercentileRanksTests extends AbstractNumericTests { private static double[] randomPercents(long minValue, long maxValue) { @@ -256,6 +258,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTests { @Override @Test + @AwaitsFix(bugUrl="Fails with seed: B75FCDC119D90BBE, Colin to fix") public void testSingleValuedField_WithValueScript_WithParams() throws Exception { int sigDigits = randomSignificantDigits(); Map params = new HashMap<>(); @@ -369,6 +372,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTests { @Override @Test + @AwaitsFix(bugUrl="Fails with seed: B75FCDC119D90BBE, Colin to fix") public void testScript_SingleValued_WithParams() throws Exception { int sigDigits = randomSignificantDigits(); Map params = new HashMap<>(); @@ -499,4 +503,4 @@ public class HDRPercentileRanksTests extends AbstractNumericTests { } } -} \ No newline at end of file +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesTests.java index a131933ecb3..c5588c06ce3 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.metrics; import com.google.common.collect.Lists; +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.script.Script; @@ -51,6 +52,7 @@ import static org.hamcrest.Matchers.sameInstance; /** * */ +@AwaitsFix(bugUrl = "single test methods fail with occassional seeds (see testScript_ExplicitSingleValued_WithParams for example) but only if run as a whole test class not if run as a single test method") public class HDRPercentilesTests extends AbstractNumericTests { private static double[] randomPercentiles() { @@ -377,6 +379,7 @@ public class HDRPercentilesTests extends AbstractNumericTests { @Override @Test + @AwaitsFix(bugUrl = "fails with -Dtests.seed=5BFFA768633A0A59 but only if run as a whole test class not if run as a single test method") public void testScript_ExplicitSingleValued_WithParams() throws Exception { Map params = new HashMap<>(); params.put("dec", 1); diff --git a/dev-tools/src/main/resources/forbidden/all-signatures.txt b/dev-tools/src/main/resources/forbidden/all-signatures.txt index cb4b2a43632..d33c4ee9b8b 100644 --- a/dev-tools/src/main/resources/forbidden/all-signatures.txt +++ b/dev-tools/src/main/resources/forbidden/all-signatures.txt @@ -60,4 +60,6 @@ com.google.common.collect.Iterators#emptyIterator() @ Use Collections.emptyItera java.io.ObjectOutputStream java.io.ObjectOutput java.io.ObjectInputStream -java.io.ObjectInput \ No newline at end of file +java.io.ObjectInput + +java.nio.file.Files#isHidden(java.nio.file.Path) @ Dependent on the operating system, use FileSystemUtils.isHidden instead diff --git a/distribution/deb/pom.xml b/distribution/deb/pom.xml new file mode 100644 index 00000000000..f4cfd05efce --- /dev/null +++ b/distribution/deb/pom.xml @@ -0,0 +1,280 @@ + + + 4.0.0 + + org.elasticsearch.distribution + elasticsearch-distribution + 2.0.0-beta1-SNAPSHOT + + + elasticsearch-deb + deb + Elasticsearch DEB Distribution + + + true + false + dpkg-sig + + + + + + ${project.basedir}/src/main/packaging/packaging.properties + + + + + org.apache.maven.plugins + maven-resources-plugin + + + + copy-resources-deb + prepare-package + + copy-resources + + + ${project.build.directory}/generated-packaging/deb/ + + ${project.basedir}/../src/main/packaging/packaging.properties + ${project.basedir}/src/main/packaging/packaging.properties + + + + ${project.basedir}/../src/main/packaging/ + true + + **/* + + + packaging.properties + + + + ${project.basedir}/src/main/packaging/ + true + + **/* + + + packaging.properties + + + + ${project.basedir}/../src/main/resources + true + + bin/elasticsearch + bin/elasticsearch.in.sh + bin/plugin + + + + + + + + + + + jdeb + org.vafer + 1.4 + + + ${project.build.directory}/releases/elasticsearch-${project.version}.deb + ${project.build.directory}/generated-packaging/deb/scripts + + + + package + + jdeb + + + ${deb.sign} + ${gpg.keyring} + ${gpg.key} + ${gpg.passphrase} + ${deb.sign.method} + + + + ${project.build.directory}/generated-packaging/deb/bin + directory + elasticsearch,elasticsearch.in.sh,plugin + + perm + ${packaging.elasticsearch.bin.dir} + 755 + root + root + + + + + ${project.basedir}/../src/main/resources/config + directory + .DS_Store + + perm + ${packaging.elasticsearch.conf.dir} + root + root + + + + + ${project.build.directory}/generated-packaging/deb/env/elasticsearch + file + + perm + /etc/default + 644 + root + root + + + + + ${project.build.directory}/ + elasticsearch-${project.version}.jar + directory + + perm + ${packaging.elasticsearch.home.dir}/lib + root + root + + + + ${project.build.directory}/../target/lib + ${project.build.finalName}-shaded.jar,${project.build.finalName}-sources.jar,${project.build.finalName}-tests.jar,${project.build.finalName}-test-sources.jar,slf4j-api-*.jar + directory + + perm + ${packaging.elasticsearch.home.dir}/lib + root + root + + + + + ${project.build.directory}/generated-packaging/deb/init.d/ + directory + .DS_Store + + perm + /etc/init.d + 755 + root + root + + + + + ${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.service + ${packaging.elasticsearch.systemd.dir}/elasticsearch.service + file + + + + ${project.build.directory}/generated-packaging/deb/systemd/sysctl/elasticsearch.conf + ${packaging.elasticsearch.systemd.sysctl.dir}/elasticsearch.conf + file + + + + ${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.conf + ${packaging.elasticsearch.tmpfilesd.dir}/elasticsearch.conf + file + + + + ${project.build.directory}/generated-packaging/deb/lintian + directory + .DS_Store + + perm + /usr/share/lintian/overrides + root + root + + + + + ${project.basedir}/../src/main/resources/ + *.txt, *.textile + LICENSE.txt, .DS_Store + directory + + perm + ${packaging.elasticsearch.home.dir} + root + root + + + + + ${project.build.directory}/generated-packaging/deb/copyright + /usr/share/doc/elasticsearch/copyright + file + + + + template + + ${packaging.elasticsearch.data.dir} + ${packaging.elasticsearch.log.dir} + ${packaging.elasticsearch.plugins.dir} + ${packaging.elasticsearch.pid.dir} + + + perm + 755 + ${packaging.elasticsearch.user} + ${packaging.elasticsearch.group} + + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + check-license + verify + + run + + + + + + + Running license check + + + + + + + + + + + + + + + diff --git a/core/src/packaging/deb/copyright b/distribution/deb/src/main/packaging/copyright similarity index 100% rename from core/src/packaging/deb/copyright rename to distribution/deb/src/main/packaging/copyright diff --git a/core/src/packaging/deb/init.d/elasticsearch b/distribution/deb/src/main/packaging/init.d/elasticsearch similarity index 97% rename from core/src/packaging/deb/init.d/elasticsearch rename to distribution/deb/src/main/packaging/init.d/elasticsearch index 0c5e91d83bf..19198c91314 100755 --- a/core/src/packaging/deb/init.d/elasticsearch +++ b/distribution/deb/src/main/packaging/init.d/elasticsearch @@ -173,7 +173,7 @@ case "$1" in fi # Start Daemon - start-stop-daemon --start -b --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS + start-stop-daemon -d $ES_HOME --start -b --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS return=$? if [ $return -eq 0 ] then diff --git a/core/src/packaging/deb/lintian/elasticsearch b/distribution/deb/src/main/packaging/lintian/elasticsearch similarity index 100% rename from core/src/packaging/deb/lintian/elasticsearch rename to distribution/deb/src/main/packaging/lintian/elasticsearch diff --git a/core/src/packaging/deb/packaging.properties b/distribution/deb/src/main/packaging/packaging.properties similarity index 100% rename from core/src/packaging/deb/packaging.properties rename to distribution/deb/src/main/packaging/packaging.properties diff --git a/core/src/packaging/deb/scripts/conffiles b/distribution/deb/src/main/packaging/scripts/conffiles similarity index 100% rename from core/src/packaging/deb/scripts/conffiles rename to distribution/deb/src/main/packaging/scripts/conffiles diff --git a/core/src/packaging/deb/scripts/control b/distribution/deb/src/main/packaging/scripts/control similarity index 100% rename from core/src/packaging/deb/scripts/control rename to distribution/deb/src/main/packaging/scripts/control diff --git a/core/licenses/HdrHistogram-2.1.5.jar.sha1 b/distribution/licenses/HdrHistogram-2.1.5.jar.sha1 similarity index 100% rename from core/licenses/HdrHistogram-2.1.5.jar.sha1 rename to distribution/licenses/HdrHistogram-2.1.5.jar.sha1 diff --git a/core/licenses/HdrHistogram-LICENSE.txt b/distribution/licenses/HdrHistogram-LICENSE.txt similarity index 100% rename from core/licenses/HdrHistogram-LICENSE.txt rename to distribution/licenses/HdrHistogram-LICENSE.txt diff --git a/core/licenses/HdrHistogram-NOTICE.txt b/distribution/licenses/HdrHistogram-NOTICE.txt similarity index 100% rename from core/licenses/HdrHistogram-NOTICE.txt rename to distribution/licenses/HdrHistogram-NOTICE.txt diff --git a/core/licenses/antlr-runtime-3.5.jar.sha1 b/distribution/licenses/antlr-runtime-3.5.jar.sha1 similarity index 100% rename from core/licenses/antlr-runtime-3.5.jar.sha1 rename to distribution/licenses/antlr-runtime-3.5.jar.sha1 diff --git a/core/licenses/antlr-runtime-LICENSE.txt b/distribution/licenses/antlr-runtime-LICENSE.txt similarity index 100% rename from core/licenses/antlr-runtime-LICENSE.txt rename to distribution/licenses/antlr-runtime-LICENSE.txt diff --git a/core/licenses/antlr-runtime-NOTICE.txt b/distribution/licenses/antlr-runtime-NOTICE.txt similarity index 100% rename from core/licenses/antlr-runtime-NOTICE.txt rename to distribution/licenses/antlr-runtime-NOTICE.txt diff --git a/core/licenses/apache-log4j-extras-1.2.17.jar.sha1 b/distribution/licenses/apache-log4j-extras-1.2.17.jar.sha1 similarity index 100% rename from core/licenses/apache-log4j-extras-1.2.17.jar.sha1 rename to distribution/licenses/apache-log4j-extras-1.2.17.jar.sha1 diff --git a/core/licenses/apache-log4j-extras-DEPENDENCIES b/distribution/licenses/apache-log4j-extras-DEPENDENCIES similarity index 100% rename from core/licenses/apache-log4j-extras-DEPENDENCIES rename to distribution/licenses/apache-log4j-extras-DEPENDENCIES diff --git a/core/licenses/apache-log4j-extras-LICENSE b/distribution/licenses/apache-log4j-extras-LICENSE similarity index 100% rename from core/licenses/apache-log4j-extras-LICENSE rename to distribution/licenses/apache-log4j-extras-LICENSE diff --git a/core/licenses/apache-log4j-extras-NOTICE b/distribution/licenses/apache-log4j-extras-NOTICE similarity index 100% rename from core/licenses/apache-log4j-extras-NOTICE rename to distribution/licenses/apache-log4j-extras-NOTICE diff --git a/core/licenses/asm-4.1.jar.sha1 b/distribution/licenses/asm-4.1.jar.sha1 similarity index 100% rename from core/licenses/asm-4.1.jar.sha1 rename to distribution/licenses/asm-4.1.jar.sha1 diff --git a/core/licenses/asm-LICENSE.txt b/distribution/licenses/asm-LICENSE.txt similarity index 100% rename from core/licenses/asm-LICENSE.txt rename to distribution/licenses/asm-LICENSE.txt diff --git a/core/licenses/asm-NOTICE.txt b/distribution/licenses/asm-NOTICE.txt similarity index 100% rename from core/licenses/asm-NOTICE.txt rename to distribution/licenses/asm-NOTICE.txt diff --git a/core/licenses/asm-commons-4.1.jar.sha1 b/distribution/licenses/asm-commons-4.1.jar.sha1 similarity index 100% rename from core/licenses/asm-commons-4.1.jar.sha1 rename to distribution/licenses/asm-commons-4.1.jar.sha1 diff --git a/core/licenses/asm-commons-LICENSE.txt b/distribution/licenses/asm-commons-LICENSE.txt similarity index 100% rename from core/licenses/asm-commons-LICENSE.txt rename to distribution/licenses/asm-commons-LICENSE.txt diff --git a/core/licenses/asm-commons-NOTICE.txt b/distribution/licenses/asm-commons-NOTICE.txt similarity index 100% rename from core/licenses/asm-commons-NOTICE.txt rename to distribution/licenses/asm-commons-NOTICE.txt diff --git a/core/licenses/commons-cli-1.2.jar.sha1 b/distribution/licenses/commons-cli-1.2.jar.sha1 similarity index 100% rename from core/licenses/commons-cli-1.2.jar.sha1 rename to distribution/licenses/commons-cli-1.2.jar.sha1 diff --git a/core/licenses/commons-cli-LICENSE.txt b/distribution/licenses/commons-cli-LICENSE.txt similarity index 100% rename from core/licenses/commons-cli-LICENSE.txt rename to distribution/licenses/commons-cli-LICENSE.txt diff --git a/core/licenses/commons-cli-NOTICE.txt b/distribution/licenses/commons-cli-NOTICE.txt similarity index 100% rename from core/licenses/commons-cli-NOTICE.txt rename to distribution/licenses/commons-cli-NOTICE.txt diff --git a/core/licenses/commons-lang3-3.3.2.jar.sha1 b/distribution/licenses/commons-lang3-3.3.2.jar.sha1 similarity index 100% rename from core/licenses/commons-lang3-3.3.2.jar.sha1 rename to distribution/licenses/commons-lang3-3.3.2.jar.sha1 diff --git a/core/licenses/commons-lang3-LICENSE.txt b/distribution/licenses/commons-lang3-LICENSE.txt similarity index 100% rename from core/licenses/commons-lang3-LICENSE.txt rename to distribution/licenses/commons-lang3-LICENSE.txt diff --git a/core/licenses/commons-lang3-NOTICE.txt b/distribution/licenses/commons-lang3-NOTICE.txt similarity index 100% rename from core/licenses/commons-lang3-NOTICE.txt rename to distribution/licenses/commons-lang3-NOTICE.txt diff --git a/core/licenses/compiler-0.8.13.jar.sha1 b/distribution/licenses/compiler-0.8.13.jar.sha1 similarity index 100% rename from core/licenses/compiler-0.8.13.jar.sha1 rename to distribution/licenses/compiler-0.8.13.jar.sha1 diff --git a/core/licenses/compiler-LICENSE.txt b/distribution/licenses/compiler-LICENSE.txt similarity index 100% rename from core/licenses/compiler-LICENSE.txt rename to distribution/licenses/compiler-LICENSE.txt diff --git a/core/licenses/compiler-NOTICE.txt b/distribution/licenses/compiler-NOTICE.txt similarity index 100% rename from core/licenses/compiler-NOTICE.txt rename to distribution/licenses/compiler-NOTICE.txt diff --git a/core/licenses/compress-lzf-1.0.2.jar.sha1 b/distribution/licenses/compress-lzf-1.0.2.jar.sha1 similarity index 100% rename from core/licenses/compress-lzf-1.0.2.jar.sha1 rename to distribution/licenses/compress-lzf-1.0.2.jar.sha1 diff --git a/core/licenses/compress-lzf-LICENSE b/distribution/licenses/compress-lzf-LICENSE similarity index 100% rename from core/licenses/compress-lzf-LICENSE rename to distribution/licenses/compress-lzf-LICENSE diff --git a/core/licenses/compress-lzf-NOTICE b/distribution/licenses/compress-lzf-NOTICE similarity index 100% rename from core/licenses/compress-lzf-NOTICE rename to distribution/licenses/compress-lzf-NOTICE diff --git a/core/licenses/groovy-all-2.4.4-indy.jar.sha1 b/distribution/licenses/groovy-all-2.4.4-indy.jar.sha1 similarity index 100% rename from core/licenses/groovy-all-2.4.4-indy.jar.sha1 rename to distribution/licenses/groovy-all-2.4.4-indy.jar.sha1 diff --git a/core/licenses/groovy-all-LICENSE-ANTLR.txt b/distribution/licenses/groovy-all-LICENSE-ANTLR.txt similarity index 100% rename from core/licenses/groovy-all-LICENSE-ANTLR.txt rename to distribution/licenses/groovy-all-LICENSE-ANTLR.txt diff --git a/core/licenses/groovy-all-LICENSE-ASM.txt b/distribution/licenses/groovy-all-LICENSE-ASM.txt similarity index 100% rename from core/licenses/groovy-all-LICENSE-ASM.txt rename to distribution/licenses/groovy-all-LICENSE-ASM.txt diff --git a/core/licenses/groovy-all-LICENSE-CLI.txt b/distribution/licenses/groovy-all-LICENSE-CLI.txt similarity index 100% rename from core/licenses/groovy-all-LICENSE-CLI.txt rename to distribution/licenses/groovy-all-LICENSE-CLI.txt diff --git a/core/licenses/groovy-all-LICENSE-JSR223.txt b/distribution/licenses/groovy-all-LICENSE-JSR223.txt similarity index 100% rename from core/licenses/groovy-all-LICENSE-JSR223.txt rename to distribution/licenses/groovy-all-LICENSE-JSR223.txt diff --git a/core/licenses/groovy-all-LICENSE.txt b/distribution/licenses/groovy-all-LICENSE.txt similarity index 100% rename from core/licenses/groovy-all-LICENSE.txt rename to distribution/licenses/groovy-all-LICENSE.txt diff --git a/core/licenses/groovy-all-NOTICE.txt b/distribution/licenses/groovy-all-NOTICE.txt similarity index 100% rename from core/licenses/groovy-all-NOTICE.txt rename to distribution/licenses/groovy-all-NOTICE.txt diff --git a/core/licenses/guava-18.0.jar.sha1 b/distribution/licenses/guava-18.0.jar.sha1 similarity index 100% rename from core/licenses/guava-18.0.jar.sha1 rename to distribution/licenses/guava-18.0.jar.sha1 diff --git a/core/licenses/guava-LICENSE.txt b/distribution/licenses/guava-LICENSE.txt similarity index 100% rename from core/licenses/guava-LICENSE.txt rename to distribution/licenses/guava-LICENSE.txt diff --git a/core/licenses/guava-NOTICE.txt b/distribution/licenses/guava-NOTICE.txt similarity index 100% rename from core/licenses/guava-NOTICE.txt rename to distribution/licenses/guava-NOTICE.txt diff --git a/core/licenses/hppc-0.7.1.jar.sha1 b/distribution/licenses/hppc-0.7.1.jar.sha1 similarity index 100% rename from core/licenses/hppc-0.7.1.jar.sha1 rename to distribution/licenses/hppc-0.7.1.jar.sha1 diff --git a/core/licenses/hppc-LICENSE.txt b/distribution/licenses/hppc-LICENSE.txt similarity index 100% rename from core/licenses/hppc-LICENSE.txt rename to distribution/licenses/hppc-LICENSE.txt diff --git a/core/licenses/hppc-NOTICE.txt b/distribution/licenses/hppc-NOTICE.txt similarity index 100% rename from core/licenses/hppc-NOTICE.txt rename to distribution/licenses/hppc-NOTICE.txt diff --git a/core/licenses/jackson-LICENSE b/distribution/licenses/jackson-LICENSE similarity index 100% rename from core/licenses/jackson-LICENSE rename to distribution/licenses/jackson-LICENSE diff --git a/core/licenses/jackson-NOTICE b/distribution/licenses/jackson-NOTICE similarity index 100% rename from core/licenses/jackson-NOTICE rename to distribution/licenses/jackson-NOTICE diff --git a/core/licenses/jackson-core-2.5.3.jar.sha1 b/distribution/licenses/jackson-core-2.5.3.jar.sha1 similarity index 100% rename from core/licenses/jackson-core-2.5.3.jar.sha1 rename to distribution/licenses/jackson-core-2.5.3.jar.sha1 diff --git a/core/licenses/jackson-dataformat-cbor-2.5.3.jar.sha1 b/distribution/licenses/jackson-dataformat-cbor-2.5.3.jar.sha1 similarity index 100% rename from core/licenses/jackson-dataformat-cbor-2.5.3.jar.sha1 rename to distribution/licenses/jackson-dataformat-cbor-2.5.3.jar.sha1 diff --git a/core/licenses/jackson-dataformat-smile-2.5.3.jar.sha1 b/distribution/licenses/jackson-dataformat-smile-2.5.3.jar.sha1 similarity index 100% rename from core/licenses/jackson-dataformat-smile-2.5.3.jar.sha1 rename to distribution/licenses/jackson-dataformat-smile-2.5.3.jar.sha1 diff --git a/core/licenses/jackson-dataformat-yaml-2.5.3.jar.sha1 b/distribution/licenses/jackson-dataformat-yaml-2.5.3.jar.sha1 similarity index 100% rename from core/licenses/jackson-dataformat-yaml-2.5.3.jar.sha1 rename to distribution/licenses/jackson-dataformat-yaml-2.5.3.jar.sha1 diff --git a/core/licenses/jna-4.1.0.jar.sha1 b/distribution/licenses/jna-4.1.0.jar.sha1 similarity index 100% rename from core/licenses/jna-4.1.0.jar.sha1 rename to distribution/licenses/jna-4.1.0.jar.sha1 diff --git a/core/licenses/jna-LICENSE.txt b/distribution/licenses/jna-LICENSE.txt similarity index 100% rename from core/licenses/jna-LICENSE.txt rename to distribution/licenses/jna-LICENSE.txt diff --git a/core/licenses/jna-NOTICE.txt b/distribution/licenses/jna-NOTICE.txt similarity index 100% rename from core/licenses/jna-NOTICE.txt rename to distribution/licenses/jna-NOTICE.txt diff --git a/core/licenses/joda-convert-1.2.jar.sha1 b/distribution/licenses/joda-convert-1.2.jar.sha1 similarity index 100% rename from core/licenses/joda-convert-1.2.jar.sha1 rename to distribution/licenses/joda-convert-1.2.jar.sha1 diff --git a/core/licenses/joda-convert-LICENSE.txt b/distribution/licenses/joda-convert-LICENSE.txt similarity index 100% rename from core/licenses/joda-convert-LICENSE.txt rename to distribution/licenses/joda-convert-LICENSE.txt diff --git a/core/licenses/joda-convert-NOTICE.txt b/distribution/licenses/joda-convert-NOTICE.txt similarity index 98% rename from core/licenses/joda-convert-NOTICE.txt rename to distribution/licenses/joda-convert-NOTICE.txt index ac95834168d..dffbcf31cac 100644 --- a/core/licenses/joda-convert-NOTICE.txt +++ b/distribution/licenses/joda-convert-NOTICE.txt @@ -1,5 +1,5 @@ -============================================================================= -= NOTICE file corresponding to section 4d of the Apache License Version 2.0 = -============================================================================= -This product includes software developed by -Joda.org (http://www.joda.org/). +============================================================================= += NOTICE file corresponding to section 4d of the Apache License Version 2.0 = +============================================================================= +This product includes software developed by +Joda.org (http://www.joda.org/). diff --git a/core/licenses/joda-time-2.8.jar.sha1 b/distribution/licenses/joda-time-2.8.jar.sha1 similarity index 100% rename from core/licenses/joda-time-2.8.jar.sha1 rename to distribution/licenses/joda-time-2.8.jar.sha1 diff --git a/core/licenses/joda-time-LICENSE.txt b/distribution/licenses/joda-time-LICENSE.txt similarity index 100% rename from core/licenses/joda-time-LICENSE.txt rename to distribution/licenses/joda-time-LICENSE.txt diff --git a/core/licenses/joda-time-NOTICE.txt b/distribution/licenses/joda-time-NOTICE.txt similarity index 98% rename from core/licenses/joda-time-NOTICE.txt rename to distribution/licenses/joda-time-NOTICE.txt index ac95834168d..dffbcf31cac 100644 --- a/core/licenses/joda-time-NOTICE.txt +++ b/distribution/licenses/joda-time-NOTICE.txt @@ -1,5 +1,5 @@ -============================================================================= -= NOTICE file corresponding to section 4d of the Apache License Version 2.0 = -============================================================================= -This product includes software developed by -Joda.org (http://www.joda.org/). +============================================================================= += NOTICE file corresponding to section 4d of the Apache License Version 2.0 = +============================================================================= +This product includes software developed by +Joda.org (http://www.joda.org/). diff --git a/core/licenses/jsr166e-1.1.0.jar.sha1 b/distribution/licenses/jsr166e-1.1.0.jar.sha1 similarity index 100% rename from core/licenses/jsr166e-1.1.0.jar.sha1 rename to distribution/licenses/jsr166e-1.1.0.jar.sha1 diff --git a/core/licenses/jsr166e-LICENSE b/distribution/licenses/jsr166e-LICENSE similarity index 100% rename from core/licenses/jsr166e-LICENSE rename to distribution/licenses/jsr166e-LICENSE diff --git a/core/licenses/jsr166e-NOTICE b/distribution/licenses/jsr166e-NOTICE similarity index 100% rename from core/licenses/jsr166e-NOTICE rename to distribution/licenses/jsr166e-NOTICE diff --git a/core/licenses/jts-1.13.jar.sha1 b/distribution/licenses/jts-1.13.jar.sha1 similarity index 100% rename from core/licenses/jts-1.13.jar.sha1 rename to distribution/licenses/jts-1.13.jar.sha1 diff --git a/core/licenses/jts-LICENSE.txt b/distribution/licenses/jts-LICENSE.txt similarity index 100% rename from core/licenses/jts-LICENSE.txt rename to distribution/licenses/jts-LICENSE.txt diff --git a/core/licenses/jts-NOTICE.txt b/distribution/licenses/jts-NOTICE.txt similarity index 100% rename from core/licenses/jts-NOTICE.txt rename to distribution/licenses/jts-NOTICE.txt diff --git a/core/licenses/log4j-1.2.17.jar.sha1 b/distribution/licenses/log4j-1.2.17.jar.sha1 similarity index 100% rename from core/licenses/log4j-1.2.17.jar.sha1 rename to distribution/licenses/log4j-1.2.17.jar.sha1 diff --git a/core/licenses/log4j-LICENSE b/distribution/licenses/log4j-LICENSE similarity index 100% rename from core/licenses/log4j-LICENSE rename to distribution/licenses/log4j-LICENSE diff --git a/core/licenses/log4j-NOTICE b/distribution/licenses/log4j-NOTICE similarity index 100% rename from core/licenses/log4j-NOTICE rename to distribution/licenses/log4j-NOTICE diff --git a/core/licenses/lucene-LICENSE.txt b/distribution/licenses/lucene-LICENSE.txt similarity index 100% rename from core/licenses/lucene-LICENSE.txt rename to distribution/licenses/lucene-LICENSE.txt diff --git a/core/licenses/lucene-NOTICE.txt b/distribution/licenses/lucene-NOTICE.txt similarity index 100% rename from core/licenses/lucene-NOTICE.txt rename to distribution/licenses/lucene-NOTICE.txt diff --git a/core/licenses/lucene-analyzers-common-5.2.1.jar.sha1 b/distribution/licenses/lucene-analyzers-common-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-analyzers-common-5.2.1.jar.sha1 rename to distribution/licenses/lucene-analyzers-common-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-backward-codecs-5.2.1.jar.sha1 b/distribution/licenses/lucene-backward-codecs-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-backward-codecs-5.2.1.jar.sha1 rename to distribution/licenses/lucene-backward-codecs-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-core-5.2.1.jar.sha1 b/distribution/licenses/lucene-core-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-core-5.2.1.jar.sha1 rename to distribution/licenses/lucene-core-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-expressions-5.2.1.jar.sha1 b/distribution/licenses/lucene-expressions-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-expressions-5.2.1.jar.sha1 rename to distribution/licenses/lucene-expressions-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-grouping-5.2.1.jar.sha1 b/distribution/licenses/lucene-grouping-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-grouping-5.2.1.jar.sha1 rename to distribution/licenses/lucene-grouping-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-highlighter-5.2.1.jar.sha1 b/distribution/licenses/lucene-highlighter-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-highlighter-5.2.1.jar.sha1 rename to distribution/licenses/lucene-highlighter-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-join-5.2.1.jar.sha1 b/distribution/licenses/lucene-join-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-join-5.2.1.jar.sha1 rename to distribution/licenses/lucene-join-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-memory-5.2.1.jar.sha1 b/distribution/licenses/lucene-memory-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-memory-5.2.1.jar.sha1 rename to distribution/licenses/lucene-memory-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-misc-5.2.1.jar.sha1 b/distribution/licenses/lucene-misc-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-misc-5.2.1.jar.sha1 rename to distribution/licenses/lucene-misc-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-queries-5.2.1.jar.sha1 b/distribution/licenses/lucene-queries-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-queries-5.2.1.jar.sha1 rename to distribution/licenses/lucene-queries-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-queryparser-5.2.1.jar.sha1 b/distribution/licenses/lucene-queryparser-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-queryparser-5.2.1.jar.sha1 rename to distribution/licenses/lucene-queryparser-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-sandbox-5.2.1.jar.sha1 b/distribution/licenses/lucene-sandbox-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-sandbox-5.2.1.jar.sha1 rename to distribution/licenses/lucene-sandbox-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-spatial-5.2.1.jar.sha1 b/distribution/licenses/lucene-spatial-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-spatial-5.2.1.jar.sha1 rename to distribution/licenses/lucene-spatial-5.2.1.jar.sha1 diff --git a/core/licenses/lucene-suggest-5.2.1.jar.sha1 b/distribution/licenses/lucene-suggest-5.2.1.jar.sha1 similarity index 100% rename from core/licenses/lucene-suggest-5.2.1.jar.sha1 rename to distribution/licenses/lucene-suggest-5.2.1.jar.sha1 diff --git a/core/licenses/netty-3.10.3.Final.jar.sha1 b/distribution/licenses/netty-3.10.3.Final.jar.sha1 similarity index 100% rename from core/licenses/netty-3.10.3.Final.jar.sha1 rename to distribution/licenses/netty-3.10.3.Final.jar.sha1 diff --git a/core/licenses/netty-LICENSE.txt b/distribution/licenses/netty-LICENSE.txt similarity index 100% rename from core/licenses/netty-LICENSE.txt rename to distribution/licenses/netty-LICENSE.txt diff --git a/core/licenses/netty-NOTICE.txt b/distribution/licenses/netty-NOTICE.txt similarity index 100% rename from core/licenses/netty-NOTICE.txt rename to distribution/licenses/netty-NOTICE.txt diff --git a/core/licenses/snakeyaml-1.12.jar.sha1 b/distribution/licenses/snakeyaml-1.12.jar.sha1 similarity index 100% rename from core/licenses/snakeyaml-1.12.jar.sha1 rename to distribution/licenses/snakeyaml-1.12.jar.sha1 diff --git a/core/licenses/snakeyaml-LICENSE.txt b/distribution/licenses/snakeyaml-LICENSE.txt similarity index 100% rename from core/licenses/snakeyaml-LICENSE.txt rename to distribution/licenses/snakeyaml-LICENSE.txt diff --git a/core/licenses/snakeyaml-NOTICE.txt b/distribution/licenses/snakeyaml-NOTICE.txt similarity index 100% rename from core/licenses/snakeyaml-NOTICE.txt rename to distribution/licenses/snakeyaml-NOTICE.txt diff --git a/core/licenses/spatial4j-0.4.1.jar.sha1 b/distribution/licenses/spatial4j-0.4.1.jar.sha1 similarity index 100% rename from core/licenses/spatial4j-0.4.1.jar.sha1 rename to distribution/licenses/spatial4j-0.4.1.jar.sha1 diff --git a/core/licenses/spatial4j-LICENSE.txt b/distribution/licenses/spatial4j-LICENSE.txt similarity index 100% rename from core/licenses/spatial4j-LICENSE.txt rename to distribution/licenses/spatial4j-LICENSE.txt diff --git a/core/licenses/spatial4j-NOTICE.txt b/distribution/licenses/spatial4j-NOTICE.txt similarity index 100% rename from core/licenses/spatial4j-NOTICE.txt rename to distribution/licenses/spatial4j-NOTICE.txt diff --git a/core/licenses/t-digest-3.0.jar.sha1 b/distribution/licenses/t-digest-3.0.jar.sha1 similarity index 100% rename from core/licenses/t-digest-3.0.jar.sha1 rename to distribution/licenses/t-digest-3.0.jar.sha1 diff --git a/core/licenses/t-digest-LICENSE.txt b/distribution/licenses/t-digest-LICENSE.txt similarity index 100% rename from core/licenses/t-digest-LICENSE.txt rename to distribution/licenses/t-digest-LICENSE.txt diff --git a/core/licenses/t-digest-NOTICE.txt b/distribution/licenses/t-digest-NOTICE.txt similarity index 100% rename from core/licenses/t-digest-NOTICE.txt rename to distribution/licenses/t-digest-NOTICE.txt diff --git a/distribution/pom.xml b/distribution/pom.xml new file mode 100644 index 00000000000..dff44574334 --- /dev/null +++ b/distribution/pom.xml @@ -0,0 +1,195 @@ + + + 4.0.0 + + org.elasticsearch + elasticsearch-parent + 2.0.0-beta1-SNAPSHOT + + + org.elasticsearch.distribution + elasticsearch-distribution + pom + Elasticsearch Distribution + + + true + + /usr/share/elasticsearch + /usr/share/elasticsearch/bin + /etc/elasticsearch + /var/lib/elasticsearch + elasticsearch + elasticsearch + /var/log/elasticsearch + ${packaging.elasticsearch.home.dir}/plugins + /var/run/elasticsearch + /usr/lib/systemd/system + /usr/lib/sysctl.d + /usr/lib/tmpfiles.d + + + /usr/bin/rpmbuild + + + + + org.elasticsearch + elasticsearch + + + + + org.apache.lucene + lucene-expressions + + + com.spatial4j + spatial4j + + + com.vividsolutions + jts + + + + com.github.spullara.mustache.java + compiler + + + org.codehaus.groovy + groovy-all + indy + + + + log4j + log4j + + + + log4j + apache-log4j-extras + + + + org.slf4j + slf4j-api + + + + net.java.dev.jna + jna + + + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + test + + + + + + elasticsearch-${project.version} + + + + org.apache.maven.plugins + maven-dependency-plugin + + + prepare-package + + copy-dependencies + + + ${project.build.directory}/lib + runtime + + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-resources + prepare-package + + copy-resources + + + ${project.build.directory}/bin + + + ${project.basedir}/../src/main/resources/bin + true + + *.exe + + + + ${project.basedir}/../src/main/resources/bin + false + + *.exe + + + + + + + + + + + + + tar + zip + deb + + + + + macos_brew + + + + /usr/local/bin/rpmbuild + + + + rpm + + + + + rpm + + + + ${packaging.rpm.rpmbuild} + + + + rpm + + + + + diff --git a/distribution/rpm/pom.xml b/distribution/rpm/pom.xml new file mode 100644 index 00000000000..9cb3e29859c --- /dev/null +++ b/distribution/rpm/pom.xml @@ -0,0 +1,390 @@ + + + 4.0.0 + + org.elasticsearch.distribution + elasticsearch-distribution + 2.0.0-beta1-SNAPSHOT + + + elasticsearch-rpm + Elasticsearch RPM Distribution + rpm + + + true + + + + + + ${project.basedir}/src/main/packaging/packaging.properties + + + + + + org.apache.maven.plugins + maven-source-plugin + 2.4 + true + + + org.apache.maven.plugins + maven-resources-plugin + + + + copy-resources-rpm + prepare-package + + copy-resources + + + ${project.build.directory}/generated-packaging/rpm/ + + ${project.basedir}/../src/main/packaging/packaging.properties + ${project.basedir}/src/main/packaging/packaging.properties + + + + ${project.basedir}/../src/main/packaging/ + true + + **/* + + + packaging.properties + + + + ${project.basedir}/src/main/packaging/ + true + + **/* + + + packaging.properties + + + + ${project.basedir}/../src/main/resources + true + + bin/elasticsearch + bin/elasticsearch.in.sh + bin/plugin + + + + + + + + + + org.codehaus.mojo + rpm-maven-plugin + + false + elasticsearch + Elasticsearch + Application/Internet + Elasticsearch + /usr + src/changelog + + _unpackaged_files_terminate_build 0 + _binaries_in_noarch_packages_terminate_build 0 + + 644 + 755 + root + root + ${project.basedir}/src/main/resources/logo/elastic.gif + + + + ${packaging.elasticsearch.bin.dir}/ + 755 + + + ${project.build.directory}/generated-packaging/rpm/bin + + elasticsearch + elasticsearch.in.sh + plugin + + + + + + + ${packaging.elasticsearch.conf.dir}/ + noreplace + + + ${project.basedir}/../src/main/resources/config/ + + *.yml + + + + + + + /etc/sysconfig/ + false + noreplace + + + ${project.build.directory}/generated-packaging/rpm/env/ + + elasticsearch + + + + + + + ${packaging.elasticsearch.home.dir}/lib + + + target/lib/ + + ${project.build.finalName}-shaded.jar + ${project.build.finalName}-sources.jar + ${project.build.finalName}-tests.jar + ${project.build.finalName}-test-sources.jar + slf4j-api-*.jar + + + + + + + /etc/init.d + false + 755 + true + + + ${project.build.directory}/generated-packaging/rpm/init.d + + elasticsearch + + + + + + + ${packaging.elasticsearch.systemd.dir} + false + true + + + ${project.build.directory}/generated-packaging/rpm/systemd + + elasticsearch.service + + + + + + + ${packaging.elasticsearch.systemd.sysctl.dir} + true + + + ${project.build.directory}/generated-packaging/rpm/systemd/sysctl + + elasticsearch.conf + + + + + + + ${packaging.elasticsearch.tmpfilesd.dir} + true + + + ${project.build.directory}/generated-packaging/rpm/systemd/ + + elasticsearch.conf + + + + + + + ${packaging.elasticsearch.home.dir} + + + ${project.basedir}/../src/main/resources/ + + LICENSE.txt + NOTICE.txt + README.textile + + + + + + + ${packaging.elasticsearch.data.dir} + 755 + ${packaging.elasticsearch.user} + ${packaging.elasticsearch.group} + + + ${packaging.elasticsearch.log.dir} + 755 + ${packaging.elasticsearch.user} + ${packaging.elasticsearch.group} + + + ${packaging.elasticsearch.plugins.dir} + 755 + ${packaging.elasticsearch.user} + ${packaging.elasticsearch.group} + + + ${packaging.elasticsearch.pid.dir} + 755 + ${packaging.elasticsearch.user} + ${packaging.elasticsearch.group} + + + + ${project.build.directory}/generated-packaging/rpm/scripts/preinst + utf-8 + + + ${project.build.directory}/generated-packaging/rpm/scripts/postinst + utf-8 + + + ${project.build.directory}/generated-packaging/rpm/scripts/prerm + utf-8 + + + ${project.build.directory}/generated-packaging/rpm/scripts/postrm + utf-8 + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-rpm + package + + copy + + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + true + ${project.build.directory}/releases/ + elasticsearch-${project.version}.rpm + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + check-license + verify + + run + + + + + + + Running license check + + + + + + + + + + + + + + + + + + release + + + package.rpm + true + + + + + + org.codehaus.mojo + rpm-maven-plugin + + + attach-rpm + + attached-rpm + + + + + + + + + sign-rpm + + + rpm.sign + true + + + + + + org.codehaus.mojo + rpm-maven-plugin + + Application/Internet + ${gpg.key} + ${gpg.keyring} + + ${gpg.passphrase} + + + + + + + + diff --git a/core/src/packaging/rpm/init.d/elasticsearch b/distribution/rpm/src/main/packaging/init.d/elasticsearch similarity index 99% rename from core/src/packaging/rpm/init.d/elasticsearch rename to distribution/rpm/src/main/packaging/init.d/elasticsearch index da3e5ac8b99..2b247f933bb 100644 --- a/core/src/packaging/rpm/init.d/elasticsearch +++ b/distribution/rpm/src/main/packaging/init.d/elasticsearch @@ -107,6 +107,7 @@ start() { touch "$pidfile" && chown "$ES_USER":"$ES_GROUP" "$pidfile" fi + cd $ES_HOME echo -n $"Starting $prog: " # if not running, start it up here, usually something like "daemon $exec" daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -Des.default.path.conf=$CONF_DIR diff --git a/core/src/packaging/rpm/packaging.properties b/distribution/rpm/src/main/packaging/packaging.properties similarity index 100% rename from core/src/packaging/rpm/packaging.properties rename to distribution/rpm/src/main/packaging/packaging.properties diff --git a/distribution/rpm/src/main/resources/logo/elastic.gif b/distribution/rpm/src/main/resources/logo/elastic.gif new file mode 100755 index 00000000000..e3b20998d53 Binary files /dev/null and b/distribution/rpm/src/main/resources/logo/elastic.gif differ diff --git a/distribution/src/main/assemblies/common-bin.xml b/distribution/src/main/assemblies/common-bin.xml new file mode 100644 index 00000000000..eed959a0e10 --- /dev/null +++ b/distribution/src/main/assemblies/common-bin.xml @@ -0,0 +1,96 @@ + + + + /lib + true + + org.apache.lucene:lucene* + log4j:log4j + log4j:apache-log4j-extras + net.java.dev.jna:jna + com.spatial4j:spatial4j + com.vividsolutions:jts + org.codehaus.groovy:groovy-all + com.google.guava:guava + com.carrotsearch:hppc + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.dataformat:jackson-dataformat-smile + com.fasterxml.jackson.dataformat:jackson-dataformat-yaml + com.fasterxml.jackson.dataformat:jackson-dataformat-cbor + joda-time:joda-time + org.joda:joda-convert + io.netty:netty + com.ning:compress-lzf + com.github.spullara.mustache.java:compiler + com.tdunning:t-digest + org.apache.commons:commons-lang3 + commons-cli:commons-cli + com.twitter:jsr166e + org.hdrhistogram:HdrHistogram + + + + /lib + false + + org.elasticsearch:elasticsearch + + + + + + ../src/main/resources/config + config + + * + + + + true + ../src/main/resources/bin + bin + dos + + elasticsearch.in.bat + elasticsearch.bat + plugin.bat + service.bat + + + + false + ../src/main/resources/bin + bin + + *.exe + + + + true + ../src/main/resources/bin + bin + 0755 + 0755 + unix + + elasticsearch.in.sh + elasticsearch + plugin + + + + + + ../src/main/resources/README.textile + + + + ../src/main/resources/LICENSE.txt + + + + ../src/main/resources/NOTICE.txt + + + + diff --git a/core/src/packaging/common/env/elasticsearch b/distribution/src/main/packaging/env/elasticsearch similarity index 100% rename from core/src/packaging/common/env/elasticsearch rename to distribution/src/main/packaging/env/elasticsearch diff --git a/core/src/packaging/common/packaging.properties b/distribution/src/main/packaging/packaging.properties similarity index 100% rename from core/src/packaging/common/packaging.properties rename to distribution/src/main/packaging/packaging.properties diff --git a/core/src/packaging/common/scripts/postinst b/distribution/src/main/packaging/scripts/postinst similarity index 100% rename from core/src/packaging/common/scripts/postinst rename to distribution/src/main/packaging/scripts/postinst diff --git a/distribution/src/main/packaging/scripts/postrm b/distribution/src/main/packaging/scripts/postrm new file mode 100644 index 00000000000..1b44d3a8b38 --- /dev/null +++ b/distribution/src/main/packaging/scripts/postrm @@ -0,0 +1,119 @@ +${packaging.scripts.header} + +# +# This script is executed in the post-removal phase +# +# On Debian, +# $1=remove : indicates a removal +# $1=purge : indicates an upgrade +# +# On RedHat, +# $1=1 : indicates an new install +# $1=2 : indicates an upgrade + + + +SOURCE_ENV_FILE=true +REMOVE_DIRS=false +REMOVE_SERVICE=false +REMOVE_USER_AND_GROUP=false + +case "$1" in + + # Debian #################################################### + remove) + REMOVE_DIRS=true + REMOVE_SERVICE=true + ;; + + purge) + REMOVE_USER_AND_GROUP=true + SOURCE_ENV_FILE=false + ;; + failed-upgrade|abort-install|abort-upgrade|disappear|upgrade|disappear) + ;; + + # RedHat #################################################### + 0) + REMOVE_DIRS=true + REMOVE_SERVICE=true + REMOVE_USER_AND_GROUP=true + ;; + 2) + # If $1=1 this is an upgrade + IS_UPGRADE=true + ;; + + *) + echo "post remove script called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# Sets the default values for elasticsearch variables used in this script +ES_USER="${packaging.elasticsearch.user}" +ES_GROUP="${packaging.elasticsearch.group}" +LOG_DIR="${packaging.elasticsearch.log.dir}" +PLUGINS_DIR="${packaging.elasticsearch.plugins.dir}" +PID_DIR="${packaging.elasticsearch.pid.dir}" +DATA_DIR="${packaging.elasticsearch.data.dir}" + +# Source the default env file +if [ "$SOURCE_ENV_FILE" = "true" ]; then + ES_ENV_FILE="${packaging.env.file}" + if [ -f "$ES_ENV_FILE" ]; then + . "$ES_ENV_FILE" + fi +fi + +if [ "$REMOVE_SERVICE" = "true" ]; then + if command -v systemctl >/dev/null; then + systemctl --no-reload disable elasticsearch.service > /dev/null 2>&1 || true + fi + + if command -v chkconfig >/dev/null; then + chkconfig --del elasticsearch 2> /dev/null || true + fi + + if command -v update-rc.d >/dev/null; then + update-rc.d elasticsearch remove >/dev/null || true + fi +fi + +if [ "$REMOVE_DIRS" = "true" ]; then + + if [ -d "$LOG_DIR" ]; then + echo -n "Deleting log directory..." + rm -rf "$LOG_DIR" + echo " OK" + fi + + if [ -d "$PLUGINS_DIR" ]; then + echo -n "Deleting plugins directory..." + rm -rf "$PLUGINS_DIR" + echo " OK" + fi + + if [ -d "$PID_DIR" ]; then + echo -n "Deleting PID directory..." + rm -rf "$PID_DIR" + echo " OK" + fi + + # Delete the data directory if and only if empty + if [ -d "$DATA_DIR" ]; then + rmdir --ignore-fail-on-non-empty "$DATA_DIR" + fi +fi + +if [ "$REMOVE_USER_AND_GROUP" = "true" ]; then + if id "$ES_USER" > /dev/null 2>&1 ; then + userdel "$ES_USER" + fi + + if getent group "$ES_GROUP" > /dev/null 2>&1 ; then + groupdel "$ES_GROUP" + fi +fi + +${packaging.scripts.footer} diff --git a/core/src/packaging/common/scripts/preinst b/distribution/src/main/packaging/scripts/preinst similarity index 100% rename from core/src/packaging/common/scripts/preinst rename to distribution/src/main/packaging/scripts/preinst diff --git a/core/src/packaging/common/scripts/prerm b/distribution/src/main/packaging/scripts/prerm similarity index 100% rename from core/src/packaging/common/scripts/prerm rename to distribution/src/main/packaging/scripts/prerm diff --git a/core/src/packaging/common/systemd/elasticsearch.conf b/distribution/src/main/packaging/systemd/elasticsearch.conf similarity index 100% rename from core/src/packaging/common/systemd/elasticsearch.conf rename to distribution/src/main/packaging/systemd/elasticsearch.conf diff --git a/core/src/packaging/common/systemd/elasticsearch.service b/distribution/src/main/packaging/systemd/elasticsearch.service similarity index 97% rename from core/src/packaging/common/systemd/elasticsearch.service rename to distribution/src/main/packaging/systemd/elasticsearch.service index a4c269973bf..b857d6136bf 100644 --- a/core/src/packaging/common/systemd/elasticsearch.service +++ b/distribution/src/main/packaging/systemd/elasticsearch.service @@ -13,6 +13,8 @@ Environment=LOG_DIR=${packaging.elasticsearch.log.dir} Environment=PID_DIR=${packaging.elasticsearch.pid.dir} EnvironmentFile=-${packaging.env.file} +WorkingDirectory=${packaging.elasticsearch.home.dir} + User=${packaging.elasticsearch.user} Group=${packaging.elasticsearch.group} diff --git a/core/src/packaging/common/systemd/sysctl/elasticsearch.conf b/distribution/src/main/packaging/systemd/sysctl/elasticsearch.conf similarity index 100% rename from core/src/packaging/common/systemd/sysctl/elasticsearch.conf rename to distribution/src/main/packaging/systemd/sysctl/elasticsearch.conf diff --git a/distribution/src/main/resources/LICENSE.txt b/distribution/src/main/resources/LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/distribution/src/main/resources/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/distribution/src/main/resources/NOTICE.txt b/distribution/src/main/resources/NOTICE.txt new file mode 100644 index 00000000000..23cae9e2063 --- /dev/null +++ b/distribution/src/main/resources/NOTICE.txt @@ -0,0 +1,5 @@ +Elasticsearch +Copyright 2009-2015 Elasticsearch + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). diff --git a/distribution/src/main/resources/README.textile b/distribution/src/main/resources/README.textile new file mode 100644 index 00000000000..720f357406b --- /dev/null +++ b/distribution/src/main/resources/README.textile @@ -0,0 +1,235 @@ +h1. Elasticsearch + +h2. A Distributed RESTful Search Engine + +h3. "https://www.elastic.co/products/elasticsearch":https://www.elastic.co/products/elasticsearch + +Elasticsearch is a distributed RESTful search engine built for the cloud. Features include: + +* Distributed and Highly Available Search Engine. +** Each index is fully sharded with a configurable number of shards. +** Each shard can have one or more replicas. +** Read / Search operations performed on either one of the replica shard. +* Multi Tenant with Multi Types. +** Support for more than one index. +** Support for more than one type per index. +** Index level configuration (number of shards, index storage, ...). +* Various set of APIs +** HTTP RESTful API +** Native Java API. +** All APIs perform automatic node operation rerouting. +* Document oriented +** No need for upfront schema definition. +** Schema can be defined per type for customization of the indexing process. +* Reliable, Asynchronous Write Behind for long term persistency. +* (Near) Real Time Search. +* Built on top of Lucene +** Each shard is a fully functional Lucene index +** All the power of Lucene easily exposed through simple configuration / plugins. +* Per operation consistency +** Single document level operations are atomic, consistent, isolated and durable. +* Open Source under the Apache License, version 2 ("ALv2") + +h2. Getting Started + +First of all, DON'T PANIC. It will take 5 minutes to get the gist of what Elasticsearch is all about. + +h3. Requirements + +You need to have a recent version of Java installed. See the "Setup":http://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html#jvm-version page for more information. + +h3. Installation + +* "Download":https://www.elastic.co/downloads/elasticsearch and unzip the Elasticsearch official distribution. +* Run @bin/elasticsearch@ on unix, or @bin\elasticsearch.bat@ on windows. +* Run @curl -X GET http://localhost:9200/@. +* Start more servers ... + +h3. Indexing + +Let's try and index some twitter like information. First, let's create a twitter user, and add some tweets (the @twitter@ index will be created automatically): + +
+curl -XPUT 'http://localhost:9200/twitter/user/kimchy' -d '{ "name" : "Shay Banon" }'
+
+curl -XPUT 'http://localhost:9200/twitter/tweet/1' -d '
+{
+    "user": "kimchy",
+    "postDate": "2009-11-15T13:12:00",
+    "message": "Trying out Elasticsearch, so far so good?"
+}'
+
+curl -XPUT 'http://localhost:9200/twitter/tweet/2' -d '
+{
+    "user": "kimchy",
+    "postDate": "2009-11-15T14:12:12",
+    "message": "Another tweet, will it be indexed?"
+}'
+
+ +Now, let's see if the information was added by GETting it: + +
+curl -XGET 'http://localhost:9200/twitter/user/kimchy?pretty=true'
+curl -XGET 'http://localhost:9200/twitter/tweet/1?pretty=true'
+curl -XGET 'http://localhost:9200/twitter/tweet/2?pretty=true'
+
+ +h3. Searching + +Mmm search..., shouldn't it be elastic? +Let's find all the tweets that @kimchy@ posted: + +
+curl -XGET 'http://localhost:9200/twitter/tweet/_search?q=user:kimchy&pretty=true'
+
+ +We can also use the JSON query language Elasticsearch provides instead of a query string: + +
+curl -XGET 'http://localhost:9200/twitter/tweet/_search?pretty=true' -d '
+{
+    "query" : {
+        "match" : { "user": "kimchy" }
+    }
+}'
+
+ +Just for kicks, let's get all the documents stored (we should see the user as well): + +
+curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -d '
+{
+    "query" : {
+        "matchAll" : {}
+    }
+}'
+
+ +We can also do range search (the @postDate@ was automatically identified as date) + +
+curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -d '
+{
+    "query" : {
+        "range" : {
+            "postDate" : { "from" : "2009-11-15T13:00:00", "to" : "2009-11-15T14:00:00" }
+        }
+    }
+}'
+
+ +There are many more options to perform search, after all, it's a search product no? All the familiar Lucene queries are available through the JSON query language, or through the query parser. + +h3. Multi Tenant - Indices and Types + +Maan, that twitter index might get big (in this case, index size == valuation). Let's see if we can structure our twitter system a bit differently in order to support such large amounts of data. + +Elasticsearch supports multiple indices, as well as multiple types per index. In the previous example we used an index called @twitter@, with two types, @user@ and @tweet@. + +Another way to define our simple twitter system is to have a different index per user (note, though that each index has an overhead). Here is the indexing curl's in this case: + +
+curl -XPUT 'http://localhost:9200/kimchy/info/1' -d '{ "name" : "Shay Banon" }'
+
+curl -XPUT 'http://localhost:9200/kimchy/tweet/1' -d '
+{
+    "user": "kimchy",
+    "postDate": "2009-11-15T13:12:00",
+    "message": "Trying out Elasticsearch, so far so good?"
+}'
+
+curl -XPUT 'http://localhost:9200/kimchy/tweet/2' -d '
+{
+    "user": "kimchy",
+    "postDate": "2009-11-15T14:12:12",
+    "message": "Another tweet, will it be indexed?"
+}'
+
+ +The above will index information into the @kimchy@ index, with two types, @info@ and @tweet@. Each user will get his own special index. + +Complete control on the index level is allowed. As an example, in the above case, we would want to change from the default 5 shards with 1 replica per index, to only 1 shard with 1 replica per index (== per twitter user). Here is how this can be done (the configuration can be in yaml as well): + +
+curl -XPUT http://localhost:9200/another_user/ -d '
+{
+    "index" : {
+        "numberOfShards" : 1,
+        "numberOfReplicas" : 1
+    }
+}'
+
+ +Search (and similar operations) are multi index aware. This means that we can easily search on more than one +index (twitter user), for example: + +
+curl -XGET 'http://localhost:9200/kimchy,another_user/_search?pretty=true' -d '
+{
+    "query" : {
+        "matchAll" : {}
+    }
+}'
+
+ +Or on all the indices: + +
+curl -XGET 'http://localhost:9200/_search?pretty=true' -d '
+{
+    "query" : {
+        "matchAll" : {}
+    }
+}'
+
+ +{One liner teaser}: And the cool part about that? You can easily search on multiple twitter users (indices), with different boost levels per user (index), making social search so much simpler (results from my friends rank higher than results from friends of my friends). + +h3. Distributed, Highly Available + +Let's face it, things will fail.... + +Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replica. By default, an index is created with 5 shards and 1 replica per shard (5/1). There are many topologies that can be used, including 1/10 (improve search performance), or 20/1 (improve indexing performance, with search executed in a map reduce fashion across shards). + +In order to play with the distributed nature of Elasticsearch, simply bring more nodes up and shut down nodes. The system will continue to serve requests (make sure you use the correct http port) with the latest data indexed. + +h3. Where to go from here? + +We have just covered a very small portion of what Elasticsearch is all about. For more information, please refer to the "elastic.co":http://www.elastic.co/products/elasticsearch website. + +h3. Building from Source + +Elasticsearch uses "Maven":http://maven.apache.org for its build system. + +In order to create a distribution, simply run the @mvn clean package +-DskipTests@ command in the cloned directory. + +The distribution will be created under @target/releases@. + +See the "TESTING":TESTING.asciidoc file for more information about +running the Elasticsearch test suite. + +h3. Upgrading to Elasticsearch 1.x? + +In order to ensure a smooth upgrade process from earlier versions of Elasticsearch (< 1.0.0), it is recommended to perform a full cluster restart. Please see the "setup reference":https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html for more details on the upgrade process. + +h1. License + +
+This software is licensed under the Apache License, version 2 ("ALv2"), quoted below.
+
+Copyright 2009-2015 Elasticsearch 
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not
+use this file except in compliance with the License. You may obtain a copy of
+the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations under
+the License.
+
diff --git a/core/bin/elasticsearch b/distribution/src/main/resources/bin/elasticsearch similarity index 97% rename from core/bin/elasticsearch rename to distribution/src/main/resources/bin/elasticsearch index bd324a7b5c4..f05f3d30100 100755 --- a/core/bin/elasticsearch +++ b/distribution/src/main/resources/bin/elasticsearch @@ -57,12 +57,12 @@ # Maven will replace the project.name with elasticsearch below. If that # hasn't been done, we assume that this is not a packaged version and the # user has forgotten to run Maven to create a package. -IS_PACKAGED_VERSION='${project.artifactId}' -if [ "$IS_PACKAGED_VERSION" != "elasticsearch" ]; then +IS_PACKAGED_VERSION='${project.parent.artifactId}' +if [ "$IS_PACKAGED_VERSION" != "elasticsearch-distribution" ]; then cat >&2 << EOF Error: You must build the project with Maven or download a pre-built package before you can run Elasticsearch. See 'Building from Source' in README.textile -or visit http://www.elasticsearch.org/download to get a pre-built package. +or visit https://www.elastic.co/download to get a pre-built package. EOF exit 1 fi diff --git a/core/bin/elasticsearch-service-mgr.exe b/distribution/src/main/resources/bin/elasticsearch-service-mgr.exe similarity index 100% rename from core/bin/elasticsearch-service-mgr.exe rename to distribution/src/main/resources/bin/elasticsearch-service-mgr.exe diff --git a/core/bin/elasticsearch-service-x64.exe b/distribution/src/main/resources/bin/elasticsearch-service-x64.exe similarity index 100% rename from core/bin/elasticsearch-service-x64.exe rename to distribution/src/main/resources/bin/elasticsearch-service-x64.exe diff --git a/core/bin/elasticsearch-service-x86.exe b/distribution/src/main/resources/bin/elasticsearch-service-x86.exe similarity index 100% rename from core/bin/elasticsearch-service-x86.exe rename to distribution/src/main/resources/bin/elasticsearch-service-x86.exe diff --git a/core/bin/elasticsearch.bat b/distribution/src/main/resources/bin/elasticsearch.bat similarity index 100% rename from core/bin/elasticsearch.bat rename to distribution/src/main/resources/bin/elasticsearch.bat diff --git a/core/bin/elasticsearch.in.bat b/distribution/src/main/resources/bin/elasticsearch.in.bat similarity index 92% rename from core/bin/elasticsearch.in.bat rename to distribution/src/main/resources/bin/elasticsearch.in.bat index b25a679fc9e..fdc5d82a649 100644 --- a/core/bin/elasticsearch.in.bat +++ b/distribution/src/main/resources/bin/elasticsearch.in.bat @@ -64,6 +64,7 @@ if "%ES_GC_LOG_FILE%" == "" goto nogclog :gclog set JAVA_OPTS=%JAVA_OPTS% -XX:+PrintGCDetails set JAVA_OPTS=%JAVA_OPTS% -XX:+PrintGCTimeStamps +set JAVA_OPTS=%JAVA_OPTS% -XX:+PrintGCDateStamps set JAVA_OPTS=%JAVA_OPTS% -XX:+PrintClassHistogram set JAVA_OPTS=%JAVA_OPTS% -XX:+PrintTenuringDistribution set JAVA_OPTS=%JAVA_OPTS% -XX:+PrintGCApplicationStoppedTime @@ -88,11 +89,5 @@ set JAVA_OPTS=%JAVA_OPTS% -Dfile.encoding=UTF-8 REM Use our provided JNA always versus the system one set JAVA_OPTS=%JAVA_OPTS% -Djna.nosys=true -set CORE_CLASSPATH=%ES_HOME%/lib/${project.build.finalName}.jar;%ES_HOME%/lib/* -if "%ES_CLASSPATH%" == "" ( -set ES_CLASSPATH=%CORE_CLASSPATH% -) else ( -set ES_CLASSPATH=%ES_CLASSPATH%;%CORE_CLASSPATH% -) - +set ES_CLASSPATH=%ES_CLASSPATH%;%ES_HOME%/lib/${project.build.finalName}.jar;%ES_HOME%/lib/* set ES_PARAMS=-Delasticsearch -Des-foreground=yes -Des.path.home="%ES_HOME%" diff --git a/core/bin/elasticsearch.in.sh b/distribution/src/main/resources/bin/elasticsearch.in.sh similarity index 91% rename from core/bin/elasticsearch.in.sh rename to distribution/src/main/resources/bin/elasticsearch.in.sh index 92eab40307e..86af7224604 100644 --- a/core/bin/elasticsearch.in.sh +++ b/distribution/src/main/resources/bin/elasticsearch.in.sh @@ -1,12 +1,6 @@ #!/bin/sh -CORE_CLASSPATH="$ES_HOME/lib/${project.build.finalName}.jar:$ES_HOME/lib/*" - -if [ "x$ES_CLASSPATH" = "x" ]; then - ES_CLASSPATH="$CORE_CLASSPATH" -else - ES_CLASSPATH="$ES_CLASSPATH:$CORE_CLASSPATH" -fi +ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/lib/${project.build.finalName}.jar:$ES_HOME/lib/*" if [ "x$ES_MIN_MEM" = "x" ]; then ES_MIN_MEM=${packaging.elasticsearch.heap.min} @@ -54,6 +48,7 @@ JAVA_OPTS="$JAVA_OPTS -XX:+UseCMSInitiatingOccupancyOnly" if [ -n "$ES_GC_LOG_FILE" ]; then JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDetails" JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCTimeStamps" + JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDateStamps" JAVA_OPTS="$JAVA_OPTS -XX:+PrintClassHistogram" JAVA_OPTS="$JAVA_OPTS -XX:+PrintTenuringDistribution" JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCApplicationStoppedTime" diff --git a/core/bin/plugin b/distribution/src/main/resources/bin/plugin similarity index 100% rename from core/bin/plugin rename to distribution/src/main/resources/bin/plugin diff --git a/core/bin/plugin.bat b/distribution/src/main/resources/bin/plugin.bat similarity index 100% rename from core/bin/plugin.bat rename to distribution/src/main/resources/bin/plugin.bat diff --git a/core/bin/service.bat b/distribution/src/main/resources/bin/service.bat similarity index 100% rename from core/bin/service.bat rename to distribution/src/main/resources/bin/service.bat diff --git a/core/config/elasticsearch.yml b/distribution/src/main/resources/config/elasticsearch.yml similarity index 100% rename from core/config/elasticsearch.yml rename to distribution/src/main/resources/config/elasticsearch.yml diff --git a/core/config/logging.yml b/distribution/src/main/resources/config/logging.yml similarity index 100% rename from core/config/logging.yml rename to distribution/src/main/resources/config/logging.yml diff --git a/core/src/test/resources/packaging/scripts/20_tar_package.bats b/distribution/src/test/resources/packaging/scripts/20_tar_package.bats similarity index 100% rename from core/src/test/resources/packaging/scripts/20_tar_package.bats rename to distribution/src/test/resources/packaging/scripts/20_tar_package.bats diff --git a/core/src/test/resources/packaging/scripts/25_tar_plugins.bats b/distribution/src/test/resources/packaging/scripts/25_tar_plugins.bats similarity index 100% rename from core/src/test/resources/packaging/scripts/25_tar_plugins.bats rename to distribution/src/test/resources/packaging/scripts/25_tar_plugins.bats diff --git a/core/src/test/resources/packaging/scripts/30_deb_package.bats b/distribution/src/test/resources/packaging/scripts/30_deb_package.bats similarity index 100% rename from core/src/test/resources/packaging/scripts/30_deb_package.bats rename to distribution/src/test/resources/packaging/scripts/30_deb_package.bats diff --git a/core/src/test/resources/packaging/scripts/40_rpm_package.bats b/distribution/src/test/resources/packaging/scripts/40_rpm_package.bats similarity index 100% rename from core/src/test/resources/packaging/scripts/40_rpm_package.bats rename to distribution/src/test/resources/packaging/scripts/40_rpm_package.bats diff --git a/core/src/test/resources/packaging/scripts/50_plugins.bats b/distribution/src/test/resources/packaging/scripts/50_plugins.bats similarity index 100% rename from core/src/test/resources/packaging/scripts/50_plugins.bats rename to distribution/src/test/resources/packaging/scripts/50_plugins.bats diff --git a/core/src/test/resources/packaging/scripts/60_systemd.bats b/distribution/src/test/resources/packaging/scripts/60_systemd.bats similarity index 100% rename from core/src/test/resources/packaging/scripts/60_systemd.bats rename to distribution/src/test/resources/packaging/scripts/60_systemd.bats diff --git a/core/src/test/resources/packaging/scripts/70_sysv_initd.bats b/distribution/src/test/resources/packaging/scripts/70_sysv_initd.bats similarity index 100% rename from core/src/test/resources/packaging/scripts/70_sysv_initd.bats rename to distribution/src/test/resources/packaging/scripts/70_sysv_initd.bats diff --git a/core/src/test/resources/packaging/scripts/packaging_test_utils.bash b/distribution/src/test/resources/packaging/scripts/packaging_test_utils.bash similarity index 100% rename from core/src/test/resources/packaging/scripts/packaging_test_utils.bash rename to distribution/src/test/resources/packaging/scripts/packaging_test_utils.bash diff --git a/distribution/tar/pom.xml b/distribution/tar/pom.xml new file mode 100644 index 00000000000..ffc33c17aec --- /dev/null +++ b/distribution/tar/pom.xml @@ -0,0 +1,75 @@ + + + 4.0.0 + + org.elasticsearch.distribution + elasticsearch-distribution + 2.0.0-beta1-SNAPSHOT + + + elasticsearch-tar + Elasticsearch TAR Distribution + + + true + + + + + ${project.basedir}/../src/main/packaging/packaging.properties + + + + + org.apache.maven.plugins + maven-assembly-plugin + + false + ${project.build.directory}/releases/ + + ${project.basedir}/src/main/assemblies/targz-bin.xml + + + + + package + + single + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + check-license + verify + + run + + + + + + + Running license check + + + + + + + + + + + + + + + diff --git a/core/src/main/assemblies/targz-bin.xml b/distribution/tar/src/main/assemblies/targz-bin.xml similarity index 72% rename from core/src/main/assemblies/targz-bin.xml rename to distribution/tar/src/main/assemblies/targz-bin.xml index f342f6a0908..7a42a2de986 100644 --- a/core/src/main/assemblies/targz-bin.xml +++ b/distribution/tar/src/main/assemblies/targz-bin.xml @@ -8,6 +8,6 @@ true - src/main/assemblies/common-bin.xml + ../src/main/assemblies/common-bin.xml diff --git a/distribution/zip/pom.xml b/distribution/zip/pom.xml new file mode 100644 index 00000000000..1ca8ed117c0 --- /dev/null +++ b/distribution/zip/pom.xml @@ -0,0 +1,131 @@ + + + 4.0.0 + + org.elasticsearch.distribution + elasticsearch-distribution + 2.0.0-beta1-SNAPSHOT + + + elasticsearch-zip + Elasticsearch ZIP Distribution + + + false + + + + + org.hamcrest + hamcrest-all + test + + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + test + + + org.apache.lucene + lucene-test-framework + test + + + org.elasticsearch + elasticsearch + test + test-jar + + + org.apache.httpcomponents + httpclient + test + + + + + + ${project.basedir}/../src/main/packaging/packaging.properties + + + + + org.apache.maven.plugins + maven-assembly-plugin + + false + ${project.build.directory}/releases/ + + ${project.basedir}/src/main/assemblies/zip-bin.xml + + + + + package + + single + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + check-license + verify + + run + + + + + + + Running license check + + + + + + + + + + + + + + integ-setup + pre-integration-test + + run + + + + + + + + + + integ-teardown + post-integration-test + + run + + + + + + + + + + + + diff --git a/core/src/main/assemblies/zip-bin.xml b/distribution/zip/src/main/assemblies/zip-bin.xml similarity index 72% rename from core/src/main/assemblies/zip-bin.xml rename to distribution/zip/src/main/assemblies/zip-bin.xml index 6ac0f7a8bb2..661d5540cf7 100644 --- a/core/src/main/assemblies/zip-bin.xml +++ b/distribution/zip/src/main/assemblies/zip-bin.xml @@ -8,6 +8,6 @@ true - src/main/assemblies/common-bin.xml + ../src/main/assemblies/common-bin.xml diff --git a/core/src/test/java/org/elasticsearch/test/rest/RestIT.java b/distribution/zip/src/test/java/org/elasticsearch/test/rest/RestIT.java similarity index 100% rename from core/src/test/java/org/elasticsearch/test/rest/RestIT.java rename to distribution/zip/src/test/java/org/elasticsearch/test/rest/RestIT.java diff --git a/docs/reference/indices/warmers.asciidoc b/docs/reference/indices/warmers.asciidoc index c32391ac9b1..b8f670af73a 100644 --- a/docs/reference/indices/warmers.asciidoc +++ b/docs/reference/indices/warmers.asciidoc @@ -66,8 +66,8 @@ curl -XPUT localhost:9200/_template/template_1 -d ' }' -------------------------------------------------- -On the same level as `types` and `source`, the `query_cache` flag is supported -to enable query caching for the warmed search request. If not specified, it will +On the same level as `types` and `source`, the `request_cache` flag is supported +to enable request caching for the warmed search request. If not specified, it will use the index level configuration of query caching. [float] @@ -140,7 +140,7 @@ where Instead of `_warmer` you can also use the plural `_warmers`. -The `query_cache` parameter can be used to enable query caching for +The `request_cache` parameter can be used to enable request caching for the search request. If not specified, it will use the index level configuration of query caching. diff --git a/docs/reference/modules/indices/request_cache.asciidoc b/docs/reference/modules/indices/request_cache.asciidoc index ee68d71fa9c..b1ce608cbf2 100644 --- a/docs/reference/modules/indices/request_cache.asciidoc +++ b/docs/reference/modules/indices/request_cache.asciidoc @@ -75,7 +75,7 @@ curl -XPUT localhost:9200/my_index/_settings -d' [float] ==== Enabling caching per request -The `query_cache` query-string parameter can be used to enable or disable +The `request_cache` query-string parameter can be used to enable or disable caching on a *per-request* basis. If set, it overrides the index-level setting: [source,js] diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index 74ef55cee6e..6c7d127bc0c 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -68,7 +68,7 @@ And here is a sample response: Defaults to `query_then_fetch`. See <> for more. -`query_cache`:: +`request_cache`:: Set to `true` or `false` to enable or disable the caching of search results for requests where `size` is 0, ie @@ -85,7 +85,7 @@ And here is a sample response: terminate_after. -Out of the above, the `search_type` and the `query_cache` must be passed as +Out of the above, the `search_type` and the `request_cache` must be passed as query-string parameters. The rest of the search request should be passed within the body itself. The body content can also be passed as a REST parameter named `source`. diff --git a/plugins/cloud-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java b/plugins/cloud-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java index c23a57f51b3..debb6f41463 100644 --- a/plugins/cloud-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java +++ b/plugins/cloud-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java @@ -29,7 +29,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.ClusterAdminClient; import org.elasticsearch.cloud.azure.AbstractAzureTest; import org.elasticsearch.cloud.azure.storage.AzureStorageService; -import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage; import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.Strings; @@ -107,9 +106,9 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest { logger.info("--> creating azure repository with path [{}]", getRepositoryPath()); PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") .setType("azure").setSettings(Settings.settingsBuilder() - .put(Storage.CONTAINER, getContainerName()) - .put(Storage.BASE_PATH, getRepositoryPath()) - .put(Storage.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) + .put(Repository.CONTAINER, getContainerName()) + .put(Repository.BASE_PATH, getRepositoryPath()) + .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES) ).get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); diff --git a/pom.xml b/pom.xml index e4e9150bbcd..21158cdfd88 100644 --- a/pom.xml +++ b/pom.xml @@ -870,7 +870,7 @@ org.apache.maven.plugins maven-shade-plugin - 2.4 + 2.4.1 @@ -918,14 +918,16 @@ ${skip.integ.tests} - org.elasticsearch - elasticsearch + org.elasticsearch.distribution + elasticsearch-zip ${elasticsearch.version} zip true + + elasticsearch-${elasticsearch.version}.zip - true ${integ.deps} @@ -937,12 +939,14 @@ org.vafer jdeb 1.3 + true org.codehaus.mojo rpm-maven-plugin - 2.1.2 + 2.1.3 + true org.apache.maven.plugins @@ -1123,28 +1127,6 @@ org.eclipse.jdt.ui.text.custom_code_templates=maven-antrun-plugin 1.8 - - check-license - verify - - run - - - - - - - Running license check - - - - - - - - - - set-permgen validate @@ -1502,7 +1484,8 @@ org.eclipse.jdt.ui.text.custom_code_templates= dev-tools rest-api-spec - plugins core + distribution + plugins diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_warmer.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_warmer.json index 9800817fab9..d1dce8bd0e5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_warmer.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_warmer.json @@ -38,6 +38,10 @@ "options" : ["open","closed","none","all"], "default" : "open", "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both, in the search request to warm." + }, + "request_cache": { + "type" : "boolean", + "description" : "Specify whether the request to be wamred shoyd use the request cache, defaults to index level setting" } } },