From 4d201d14154491a0e28d477256d1e4728ef0fd80 Mon Sep 17 00:00:00 2001 From: Alexander Clare Date: Sun, 19 Jul 2015 00:49:39 -0400 Subject: [PATCH 1/9] Fix malformed query generation --- .../lucene/search/function/FieldValueFactorFunction.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java index 135cb53f65f..8aa4897a8e0 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java @@ -161,9 +161,6 @@ public class FieldValueFactorFunction extends ScoreFunction { @Override public String toString() { - if (this == NONE) { - return ""; - } return super.toString().toLowerCase(Locale.ROOT); } } From dcd3a2c56005128e5e4aa279b52f1c46b3a05b10 Mon Sep 17 00:00:00 2001 From: Alexander Clare Date: Sun, 19 Jul 2015 11:42:12 -0400 Subject: [PATCH 2/9] Test behavior of explicit Modifier.NONE --- .../functionscore/FunctionScoreFieldValueTests.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueTests.java b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueTests.java index eef4ed27959..2a780fd9fec 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueTests.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreFieldValueTests.java @@ -69,6 +69,14 @@ public class FunctionScoreFieldValueTests extends ElasticsearchIntegrationTest { .get(); assertOrderedSearchHits(response, "2", "1"); + // try again, but this time explicitly use the do-nothing modifier + response = client().prepareSearch("test") + .setExplain(randomBoolean()) + .setQuery(functionScoreQuery(simpleQueryStringQuery("foo"), + fieldValueFactorFunction("test").modifier(FieldValueFactorFunction.Modifier.NONE))) + .get(); + assertOrderedSearchHits(response, "2", "1"); + // document 1 scores higher because 1/5 > 1/17 response = client().prepareSearch("test") .setExplain(randomBoolean()) From 3fd7a36f91c035b4c00f2615f95929d54aab0695 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 21 Jul 2015 15:32:31 +0200 Subject: [PATCH 3/9] Remove the dependecy on IndexFielddataService from MapperService. This dependency was used in order for mapping updates that change the fielddata format to take effect immediately. And the way it worked was by clearing the cache of fielddata instances that were already loaded. However, we do not need to cache the already loaded (logical) fielddata instances, they are cheap to regenerate. Note that the fielddata _caches_ are still kept around so that we don't keep on rebuilding costly (physical) fielddata values. --- .../fielddata/IndexFieldDataService.java | 121 ++++++------------ .../index/mapper/MapperService.java | 6 +- .../index/engine/InternalEngineTests.java | 3 +- .../fielddata/IndexFieldDataServiceTests.java | 1 - 4 files changed, 44 insertions(+), 87 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index f851420e036..287d18c6051 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.KeyedLock; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.Index; @@ -44,7 +43,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentMap; import static org.elasticsearch.index.mapper.MappedFieldType.Names; @@ -138,7 +136,6 @@ public class IndexFieldDataService extends AbstractIndexComponent { } private final IndicesFieldDataCache indicesFieldDataCache; - private final ConcurrentMap> loadedFieldData = ConcurrentCollections.newConcurrentMap(); private final KeyedLock.GlobalLockable fieldLoadingLock = new KeyedLock.GlobalLockable<>(); private final Map fieldDataCaches = Maps.newHashMap(); // no need for concurrency support, always used under lock @@ -161,15 +158,6 @@ public class IndexFieldDataService extends AbstractIndexComponent { fieldLoadingLock.globalLock().lock(); try { List exceptions = new ArrayList<>(0); - final Collection> fieldDataValues = loadedFieldData.values(); - for (IndexFieldData fieldData : fieldDataValues) { - try { - fieldData.clear(); - } catch (Throwable t) { - exceptions.add(t); - } - } - fieldDataValues.clear(); final Collection fieldDataCacheValues = fieldDataCaches.values(); for (IndexFieldDataCache cache : fieldDataCacheValues) { try { @@ -189,14 +177,6 @@ public class IndexFieldDataService extends AbstractIndexComponent { fieldLoadingLock.acquire(fieldName); try { List exceptions = new ArrayList<>(0); - final IndexFieldData fieldData = loadedFieldData.remove(fieldName); - if (fieldData != null) { - try { - fieldData.clear(); - } catch (Throwable t) { - exceptions.add(t); - } - } final IndexFieldDataCache cache = fieldDataCaches.remove(fieldName); if (cache != null) { try { @@ -211,17 +191,6 @@ public class IndexFieldDataService extends AbstractIndexComponent { } } - public void onMappingUpdate() { - // synchronize to make sure to not miss field data instances that are being loaded - fieldLoadingLock.globalLock().lock(); - try { - // important: do not clear fieldDataCaches: the cache may be reused - loadedFieldData.clear(); - } finally { - fieldLoadingLock.globalLock().unlock(); - } - } - @SuppressWarnings("unchecked") public > IFD getForField(MappedFieldType fieldType) { final Names fieldNames = fieldType.names(); @@ -231,57 +200,49 @@ public class IndexFieldDataService extends AbstractIndexComponent { } final boolean docValues = fieldType.hasDocValues(); final String key = fieldNames.indexName(); - IndexFieldData fieldData = loadedFieldData.get(key); - if (fieldData == null) { - fieldLoadingLock.acquire(key); - try { - fieldData = loadedFieldData.get(key); - if (fieldData == null) { - IndexFieldData.Builder builder = null; - String format = type.getFormat(indexSettings); - if (format != null && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(format) && !docValues) { - logger.warn("field [" + fieldNames.fullName() + "] has no doc values, will use default field data format"); - format = null; - } - if (format != null) { - builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), format)); - if (builder == null) { - logger.warn("failed to find format [" + format + "] for field [" + fieldNames.fullName() + "], will use default"); - } - } - if (builder == null && docValues) { - builder = docValuesBuildersByType.get(type.getType()); - } - if (builder == null) { - builder = buildersByType.get(type.getType()); - } - if (builder == null) { - throw new IllegalArgumentException("failed to find field data builder for field " + fieldNames.fullName() + ", and type " + type.getType()); - } - - IndexFieldDataCache cache = fieldDataCaches.get(fieldNames.indexName()); - if (cache == null) { - // we default to node level cache, which in turn defaults to be unbounded - // this means changing the node level settings is simple, just set the bounds there - String cacheType = type.getSettings().get("cache", indexSettings.get(FIELDDATA_CACHE_KEY, FIELDDATA_CACHE_VALUE_NODE)); - if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) { - cache = indicesFieldDataCache.buildIndexFieldDataCache(indexService, index, fieldNames, type); - } else if ("none".equals(cacheType)){ - cache = new IndexFieldDataCache.None(); - } else { - throw new IllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]"); - } - fieldDataCaches.put(fieldNames.indexName(), cache); - } - - fieldData = builder.build(index, indexSettings, fieldType, cache, circuitBreakerService, indexService.mapperService()); - loadedFieldData.put(fieldNames.indexName(), fieldData); - } - } finally { - fieldLoadingLock.release(key); + fieldLoadingLock.acquire(key); + try { + IndexFieldData.Builder builder = null; + String format = type.getFormat(indexSettings); + if (format != null && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(format) && !docValues) { + logger.warn("field [" + fieldNames.fullName() + "] has no doc values, will use default field data format"); + format = null; } + if (format != null) { + builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), format)); + if (builder == null) { + logger.warn("failed to find format [" + format + "] for field [" + fieldNames.fullName() + "], will use default"); + } + } + if (builder == null && docValues) { + builder = docValuesBuildersByType.get(type.getType()); + } + if (builder == null) { + builder = buildersByType.get(type.getType()); + } + if (builder == null) { + throw new IllegalArgumentException("failed to find field data builder for field " + fieldNames.fullName() + ", and type " + type.getType()); + } + + IndexFieldDataCache cache = fieldDataCaches.get(fieldNames.indexName()); + if (cache == null) { + // we default to node level cache, which in turn defaults to be unbounded + // this means changing the node level settings is simple, just set the bounds there + String cacheType = type.getSettings().get("cache", indexSettings.get(FIELDDATA_CACHE_KEY, FIELDDATA_CACHE_VALUE_NODE)); + if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) { + cache = indicesFieldDataCache.buildIndexFieldDataCache(indexService, index, fieldNames, type); + } else if ("none".equals(cacheType)){ + cache = new IndexFieldDataCache.None(); + } else { + throw new IllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]"); + } + fieldDataCaches.put(fieldNames.indexName(), cache); + } + + return (IFD) builder.build(index, indexSettings, fieldType, cache, circuitBreakerService, indexService.mapperService()); + } finally { + fieldLoadingLock.release(key); } - return (IFD) fieldData; } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index cf5d9f4edbd..8f20ef5f7ab 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -53,7 +53,6 @@ import org.elasticsearch.common.util.concurrent.ReleasableLock; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.Index; import org.elasticsearch.index.analysis.AnalysisService; -import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.Mapper.BuilderContext; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; @@ -104,7 +103,6 @@ public class MapperService extends AbstractIndexComponent { }; private final AnalysisService analysisService; - private final IndexFieldDataService fieldDataService; /** * Will create types automatically if they do not exists in the mapping definition yet @@ -139,12 +137,11 @@ public class MapperService extends AbstractIndexComponent { private volatile ImmutableSet parentTypes = ImmutableSet.of(); @Inject - public MapperService(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService, IndexFieldDataService fieldDataService, + public MapperService(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService, SimilarityLookupService similarityLookupService, ScriptService scriptService) { super(index, indexSettings); this.analysisService = analysisService; - this.fieldDataService = fieldDataService; this.fieldTypes = new FieldTypeLookup(); this.documentParser = new DocumentMapperParser(indexSettings, this, analysisService, similarityLookupService, scriptService); this.indexAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultIndexAnalyzer(), INDEX_ANALYZER_EXTRACTOR); @@ -291,7 +288,6 @@ public class MapperService extends AbstractIndexComponent { logger.debug("merging mapping for type [{}] resulted in conflicts: [{}]", mapper.type(), Arrays.toString(result.buildConflicts())); } } - fieldDataService.onMappingUpdate(); return oldMapper; } else { List newObjectMappers = new ArrayList<>(); diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 6856eb9d0ea..45a730a50f0 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -101,6 +101,7 @@ import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY; import static org.elasticsearch.index.engine.Engine.Operation.Origin.REPLICA; import static org.hamcrest.Matchers.*; + public class InternalEngineTests extends ElasticsearchTestCase { private static final Pattern PARSE_LEGACY_ID_PATTERN = Pattern.compile("^" + Translog.TRANSLOG_FILE_PREFIX + "(\\d+)((\\.recovering))?$"); @@ -1923,7 +1924,7 @@ public class InternalEngineTests extends ElasticsearchTestCase { Index index = new Index(indexName); AnalysisService analysisService = new AnalysisService(index, settings); SimilarityLookupService similarityLookupService = new SimilarityLookupService(index, settings); - MapperService mapperService = new MapperService(index, settings, analysisService, null, similarityLookupService, null); + MapperService mapperService = new MapperService(index, settings, analysisService, similarityLookupService, null); DocumentMapper.Builder b = new DocumentMapper.Builder(settings, rootBuilder, mapperService); DocumentMapperParser parser = new DocumentMapperParser(settings, mapperService, analysisService, similarityLookupService, null); this.docMapper = b.build(mapperService, parser); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 3af20da91ed..2823566fddd 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -150,7 +150,6 @@ public class IndexFieldDataServiceTests extends ElasticsearchSingleNodeTest { writer.addDocument(doc); final IndexReader reader2 = DirectoryReader.open(writer, true); final MappedFieldType mapper2 = MapperBuilders.stringField("s").tokenized(false).docValues(true).fieldDataSettings(Settings.builder().put(FieldDataType.FORMAT_KEY, "doc_values").build()).build(ctx).fieldType(); - ifdService.onMappingUpdate(); ifd = ifdService.getForField(mapper2); assertThat(ifd, instanceOf(SortedSetDVOrdinalsIndexFieldData.class)); reader1.close(); From 4040f194f528c76c88be70b119cf9d7d6fcea99b Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 22 Jul 2015 10:44:52 -0400 Subject: [PATCH 4/9] Refactor pluginservice Closes #12367 Squashed commit of the following: commit 9453c411798121aa5439c52e95301f60a022ba5f Merge: 3511a9c 828d8c7 Author: Robert Muir Date: Wed Jul 22 08:22:41 2015 -0400 Merge branch 'master' into refactor_pluginservice commit 3511a9c616503c447de9f0df9b4e9db3e22abd58 Author: Ryan Ernst Date: Tue Jul 21 21:50:15 2015 -0700 Remove duplicated constant commit 4a9b5b4621b0ef2e74c1e017d9c8cf624dd27713 Author: Ryan Ernst Date: Tue Jul 21 21:01:57 2015 -0700 Add check that plugin must specify at least site or jvm commit 19aef2f0596153a549ef4b7f4483694de41e101b Author: Ryan Ernst Date: Tue Jul 21 20:52:58 2015 -0700 Change plugin "plugin" property to "classname" commit 07ae396f30ed592b7499a086adca72d3f327fe4c Author: Robert Muir Date: Tue Jul 21 23:36:05 2015 -0400 remove test with no methods commit 550e73bf3d0f94562f4dde95239409dc5a24ce25 Author: Robert Muir Date: Tue Jul 21 23:31:58 2015 -0400 fix loading to use classname commit 04463aed12046da0da5cac2a24c3ace51a79f799 Author: Robert Muir Date: Tue Jul 21 23:24:19 2015 -0400 rename to classname commit 9f3afadd1caf89448c2eb913757036da48758b2d Author: Ryan Ernst Date: Tue Jul 21 20:18:46 2015 -0700 moved PluginInfo and refactored parsing from properties file commit df63ccc1b8b7cc64d3e59d23f6c8e827825eba87 Author: Robert Muir Date: Tue Jul 21 23:08:26 2015 -0400 fix test commit c7febd844be358707823186a8c7a2d21e37540c9 Author: Robert Muir Date: Tue Jul 21 23:03:44 2015 -0400 remove test commit 017b3410cf9d2b7fca1b8653e6f1ebe2f2519257 Author: Robert Muir Date: Tue Jul 21 22:58:31 2015 -0400 fix test commit c9922938df48041ad43bbb3ed6746f71bc846629 Merge: ad59af4 01ea89a Author: Robert Muir Date: Tue Jul 21 22:37:28 2015 -0400 Merge branch 'master' into refactor_pluginservice commit ad59af465e1f1ac58897e63e0c25fcce641148a7 Author: Areek Zillur Date: Tue Jul 21 19:30:26 2015 -0400 [TEST] Verify expected number of nodes in cluster before issuing shardStores request commit f0f5a1e087255215b93656550fbc6bd89b8b3205 Author: Lee Hinman Date: Tue Jul 21 11:27:28 2015 -0600 Ignore EngineClosedException during translog fysnc When performing an operation on a primary, the state is captured and the operation is performed on the primary shard. The original request is then modified to increment the version of the operation as preparation for it to be sent to the replicas. If the request first fails on the primary during the translog sync (because the Engine is already closed due to shadow primaries closing the engine on relocation), then the operation is retried on the new primary after being modified for the replica shards. It will then fail due to the version being incorrect (the document does not yet exist but the request expects a version of "1"). Order of operations: - Request is executed against primary - Request is modified (version incremented) so it can be sent to replicas - Engine's translog is fsync'd if necessary (failing, and throwing an exception) - Modified request is retried against new primary This change ignores the exception where the engine is already closed when syncing the translog (similar to how we ignore exceptions when refreshing the shard if the ?refresh=true flag is used). commit 4ac68bb1658688550ced0c4f479dee6d8b617777 Author: Shay Banon Date: Tue Jul 21 22:37:29 2015 +0200 Replica allocator unit tests First batch of unit tests to verify the behavior of replica allocator commit 94609fc5943c8d85adc751b553847ab4cebe58a3 Author: Jason Tedor Date: Tue Jul 21 14:04:46 2015 -0400 Correctly list blobs in Azure storage to prevent snapshot corruption and do not unnecessarily duplicate Lucene segments in Azure Storage This commit addresses an issue that was leading to snapshot corruption for snapshots stored as blobs in Azure Storage. The underlying issue is that in cases when multiple snapshots of an index were taken and persisted into Azure Storage, snapshots subsequent to the first would repeatedly overwrite the snapshot files. This issue does render useless all snapshots except the final snapshot. The root cause of this is due to String concatenation involving null. In particular, to list all of the blobs in a snapshot directory in Azure the code would use the method listBlobsByPrefix where the prefix is null. In the listBlobsByPrefix method, the path keyPath + prefix is constructed. However, per 5.1.11, 5.4 and 15.18.1 of the Java Language Specification, the reference null is first converted to the string "null" before performing the concatenation. This leads to no blobs being returned and therefore the snapshot mechanism would operate as if it were writing the first snapshot of the index. The fix is simply to check if prefix is null and handle the concatenation accordingly. Upon fixing this issue so that subsequent snapshots would no longer overwrite earlier snapshots, it was discovered that the snapshot metadata returned by the listBlobsByPrefix method was not sufficient for the snapshot layer to detect whether or not the Lucene segments had already been copied to the Azure storage layer in an earlier snapshot. This led the snapshot layer to unnecessarily duplicate these Lucene segments in Azure Storage. The root cause of this is due to known behavior in the CloudBlobContainer.getBlockBlobReference method in the Azure API. Namely, this method does not fetch blob attributes from Azure. As such, the lengths of all the blobs appeared to the snapshot layer to be of length zero and therefore they would compare as not equal to any new blobs that the snapshot layer is going to persist. To remediate this, the method CloudBlockBlob.downloadAttributes must be invoked. This will fetch the attributes from Azure Storage so that a proper comparison of the blobs can be performed. Closes elastic/elasticsearch-cloud-azure#51, closes elastic/elasticsearch-cloud-azure#99 commit cf1d481ce5dda0a45805e42f3b2e0e1e5d028b9e Author: Lee Hinman Date: Mon Jul 20 08:41:55 2015 -0600 Unit tests for `nodesAndVersions` on shared filesystems With the `recover_on_any_node` setting, these unit tests check that the correct node list and versions are returned. commit 3c27cc32395c3624f7c794904d9ea4faf2eccbfb Author: Robert Muir Date: Tue Jul 21 14:15:59 2015 -0400 don't fail junit4 integration tests if there are no tests. instead fail the failsafe plugin, which means the external cluster will still get shut down commit 95d2756c5a8c21a157fa844273fc83dfa3c00aea Author: Alexander Reelsen Date: Tue Jul 21 17:16:53 2015 +0200 Testing: Fix help displaying tests under windows The help files are using a unix based file separator, where as the test relies on the help being based on the file system separator. This commit fixes the test to remove all `\r` characters before comparing strings. The test has also been moved into its own CliToolTestCase, as it does not need to be an integration test. commit 944f06ea36bd836f007f8eaade8f571d6140aad9 Author: Clinton Gormley Date: Tue Jul 21 18:04:52 2015 +0200 Refactored check_license_and_sha.pl to accept a license dir and package path In preparation for the move to building the core zip, tar.gz, rpm, and deb as separate modules, refactored check_license_and_sha.pl to: * accept a license dir and path to the package to check on the command line * to be able to extract zip, tar.gz, deb, and rpm * all packages except rpm will work on Windows commit 2585431e8dfa5c82a2cc5b304cd03eee9bed7a4c Author: Chris Earle Date: Tue Jul 21 08:35:28 2015 -0700 Updating breaking changes - field names cannot be mapped with `.` in them - fixed asciidoc issue where the list was not recognized as a list commit de299b9d3f4615b12e2226a1e2eff5a38ecaf15f Author: Shay Banon Date: Tue Jul 21 13:27:52 2015 +0200 Replace primaryPostAllocated flag and use UnassignedInfo There is no need to maintain additional state as to if a primary was allocated post api creation on the index routing table, we hold all this information already in the UnassignedInfo class. closes #12374 commit 43080bff40f60bedce5bdbc92df302f73aeb9cae Author: Alexander Reelsen Date: Tue Jul 21 15:45:05 2015 +0200 PluginManager: Fix bin/plugin calls in scripts/bats test The release and smoke test python scripts used to install plugins in the old fashion. Also the BATS testing suite installed/removed plugins in that way. Here the marvel tests have been removed, as marvel currently does not work with the master branch. In addition documentation has been updated as well, where it was still missing. commit b81ccba48993bc13c7678e6d979fd96998499233 Author: Boaz Leskes Date: Tue Jul 21 11:37:50 2015 +0200 Discovery: make sure NodeJoinController.ElectionCallback is always called from the update cluster state thread This is important for correct handling of the joining thread. This causes assertions to trip in our test runs. See http://build-us-00.elastic.co/job/es_g1gc_master_metal/11653/ as an example Closes #12372 commit 331853790bf29e34fb248ebc4c1ba585b44f5cab Author: Boaz Leskes Date: Tue Jul 21 15:54:36 2015 +0200 Remove left over no commit from TransportReplicationAction It asks to double check thread pool rejection. I did and don't see problems with it. commit e5724931bbc1603e37faa977af4235507f4811f5 Author: Alexander Reelsen Date: Tue Jul 21 15:31:57 2015 +0200 CliTool: Various PluginManager fixes The new plugin manager parser was not called correctly in the scripts. In addition the plugin manager now creates a plugins/ directory in case it does not exist. Also the integration tests called the plugin manager in the deprecated way. commit 7a815a370f83ff12ffb12717ac2fe62571311279 Author: Alexander Reelsen Date: Tue Jul 21 13:54:18 2015 +0200 CLITool: Port PluginManager to use CLITool In order to unify the handling and reuse the CLITool infrastructure the plugin manager should make use of this as well. This obsolets the -i and --install options but requires the user to use `install` as the first argument of the CLI. This is basically just a port of the existing functionality, which is also the reason why this is not a refactoring of the plugin manager, which will come in a separate commit. commit 7f171eba7b71ac5682a355684b6da703ffbfccc7 Author: Martijn van Groningen Date: Tue Jul 21 10:44:21 2015 +0200 Remove custom execute local logic in TransportSingleShardAction and TransportInstanceSingleOperationAction and rely on transport service to execute locally. (forking thread etc.) Change TransportInstanceSingleOperationAction to have shardActionHandler to, so we can execute locally without endless spinning. commit 0f38e3eca6b570f74b552e70b4673f47934442e1 Author: Ryan Ernst Date: Tue Jul 21 17:36:12 2015 -0700 More readMetadata tests and pickiness commit 880b47281bd69bd37807e8252934321b089c9f8e Author: Ryan Ernst Date: Tue Jul 21 14:42:09 2015 -0700 Started unit tests for plugin service commit cd7c8ddd7b8c4f3457824b493bffb19c156c7899 Author: Robert Muir Date: Tue Jul 21 07:21:07 2015 -0400 fix tests commit 673454f0b14f072f66ed70e32110fae4f7aad642 Author: Robert Muir Date: Tue Jul 21 06:58:25 2015 -0400 refactor pluginservice --- .../admin/cluster/node/info/PluginsInfo.java | 3 +- .../cluster/stats/ClusterStatsNodes.java | 5 +- .../elasticsearch/bootstrap/Bootstrap.java | 89 +--- .../org/elasticsearch/bootstrap/Security.java | 1 - .../node/info => plugins}/PluginInfo.java | 101 +++- .../elasticsearch/plugins/PluginsService.java | 437 ++++++------------ .../rest/action/cat/RestPluginsAction.java | 2 +- .../elasticsearch/bootstrap/security.policy | 8 +- .../cluster/node/info/PluginsInfoTest.java | 51 -- .../bootstrap/BootstrapTests.java | 47 -- .../transport/TransportClientRetryTests.java | 1 - .../nodesinfo/SimpleNodesInfoTests.java | 63 --- .../plugins/PluginInfoTests.java | 197 ++++++++ .../plugins/PluginManagerTests.java | 108 +---- .../lucene/current/CurrentLucenePlugin.java | 40 -- .../lucene/current/es-plugin-test.properties | 21 - .../lucene/newer/NewerLucenePlugin.java | 40 -- .../lucene/newer/es-plugin-test.properties | 21 - .../plugins/lucene/old/OldLucenePlugin.java | 40 -- .../lucene/old/es-plugin-test.properties | 21 - .../test/InternalTestCluster.java | 2 - .../hamcrest/ElasticsearchAssertions.java | 2 +- .../plugins/loading/jar/in-jar-plugin.jar | Bin 2039 -> 0 bytes .../plugins/plugin_folder_file.zip | Bin 503 -> 0 bytes .../plugins/plugin_folder_site.zip | Bin 373 -> 0 bytes .../plugins/plugin_single_folder.zip | Bin 405 -> 0 bytes dev-tools/smoke_test_plugins.py | 2 +- .../plugin-metadata/plugin-assembly.xml | 7 + .../plugin-descriptor.properties | 32 ++ plugins/analysis-icu/pom.xml | 1 + .../src/main/resources/es-plugin.properties | 3 - plugins/analysis-kuromoji/pom.xml | 1 + .../src/main/assemblies/plugin.xml | 19 - .../src/main/resources/es-plugin.properties | 3 - plugins/analysis-phonetic/pom.xml | 3 +- .../src/main/assemblies/plugin.xml | 19 - .../src/main/resources/es-plugin.properties | 3 - plugins/analysis-smartcn/pom.xml | 1 + .../src/main/assemblies/plugin.xml | 19 - .../src/main/resources/es-plugin.properties | 3 - plugins/analysis-stempel/pom.xml | 1 + .../src/main/assemblies/plugin.xml | 19 - .../src/main/resources/es-plugin.properties | 3 - plugins/cloud-aws/pom.xml | 1 + .../cloud-aws/src/main/assemblies/plugin.xml | 19 - .../src/main/resources/es-plugin.properties | 3 - .../cloud/aws/AbstractAwsTest.java | 3 +- .../discovery/ec2/Ec2DiscoveryITest.java | 3 +- .../ec2/Ec2DiscoveryUpdateSettingsITest.java | 3 +- .../s3/AbstractS3SnapshotRestoreTest.java | 3 +- plugins/cloud-azure/pom.xml | 2 +- .../src/main/assemblies/plugin.xml | 19 - .../src/main/resources/es-plugin.properties | 12 - .../cloud/azure/AbstractAzureTest.java | 4 +- .../AbstractAzureComputeServiceTest.java | 4 +- .../AbstractAzureRepositoryServiceTest.java | 4 +- plugins/cloud-gce/pom.xml | 1 + .../cloud-gce/src/main/assemblies/plugin.xml | 19 - .../src/main/resources/es-plugin.properties | 12 - .../cloud/gce/AbstractGceTest.java | 3 +- .../discovery/gce/GceComputeEngineTest.java | 4 +- .../gce/itest/GceSimpleITest.java | 3 +- plugins/delete-by-query/.gitignore | 1 + plugins/delete-by-query/pom.xml | 1 + .../src/main/assemblies/plugin.xml | 34 -- .../src/main/resources/es-plugin.properties | 3 - .../deletebyquery/DeleteByQueryTests.java | 7 + .../test/rest/DeleteByQueryRestTests.java | 10 + plugins/lang-javascript/pom.xml | 2 +- .../src/main/assemblies/plugin.xml | 19 - .../src/main/resources/es-plugin.properties | 2 - plugins/lang-python/pom.xml | 2 +- .../src/main/assemblies/plugin.xml | 19 - .../src/main/resources/es-plugin.properties | 2 - plugins/pom.xml | 24 +- pom.xml | 1 + 76 files changed, 563 insertions(+), 1125 deletions(-) rename core/src/main/java/org/elasticsearch/{action/admin/cluster/node/info => plugins}/PluginInfo.java (58%) delete mode 100644 core/src/test/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfoTest.java delete mode 100644 core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java create mode 100644 core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java delete mode 100644 core/src/test/java/org/elasticsearch/plugins/lucene/current/CurrentLucenePlugin.java delete mode 100644 core/src/test/java/org/elasticsearch/plugins/lucene/current/es-plugin-test.properties delete mode 100644 core/src/test/java/org/elasticsearch/plugins/lucene/newer/NewerLucenePlugin.java delete mode 100644 core/src/test/java/org/elasticsearch/plugins/lucene/newer/es-plugin-test.properties delete mode 100644 core/src/test/java/org/elasticsearch/plugins/lucene/old/OldLucenePlugin.java delete mode 100644 core/src/test/java/org/elasticsearch/plugins/lucene/old/es-plugin-test.properties delete mode 100644 core/src/test/resources/org/elasticsearch/plugins/loading/jar/in-jar-plugin.jar delete mode 100644 core/src/test/resources/org/elasticsearch/plugins/plugin_folder_file.zip delete mode 100644 core/src/test/resources/org/elasticsearch/plugins/plugin_folder_site.zip delete mode 100644 core/src/test/resources/org/elasticsearch/plugins/plugin_single_folder.zip rename plugins/analysis-icu/src/main/assemblies/plugin.xml => dev-tools/src/main/resources/plugin-metadata/plugin-assembly.xml (71%) create mode 100644 dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties delete mode 100644 plugins/analysis-icu/src/main/resources/es-plugin.properties delete mode 100644 plugins/analysis-kuromoji/src/main/assemblies/plugin.xml delete mode 100644 plugins/analysis-kuromoji/src/main/resources/es-plugin.properties delete mode 100644 plugins/analysis-phonetic/src/main/assemblies/plugin.xml delete mode 100644 plugins/analysis-phonetic/src/main/resources/es-plugin.properties delete mode 100644 plugins/analysis-smartcn/src/main/assemblies/plugin.xml delete mode 100644 plugins/analysis-smartcn/src/main/resources/es-plugin.properties delete mode 100644 plugins/analysis-stempel/src/main/assemblies/plugin.xml delete mode 100644 plugins/analysis-stempel/src/main/resources/es-plugin.properties delete mode 100644 plugins/cloud-aws/src/main/assemblies/plugin.xml delete mode 100644 plugins/cloud-aws/src/main/resources/es-plugin.properties delete mode 100644 plugins/cloud-azure/src/main/assemblies/plugin.xml delete mode 100644 plugins/cloud-azure/src/main/resources/es-plugin.properties delete mode 100644 plugins/cloud-gce/src/main/assemblies/plugin.xml delete mode 100644 plugins/cloud-gce/src/main/resources/es-plugin.properties create mode 100644 plugins/delete-by-query/.gitignore delete mode 100644 plugins/delete-by-query/src/main/assemblies/plugin.xml delete mode 100644 plugins/delete-by-query/src/main/resources/es-plugin.properties delete mode 100644 plugins/lang-javascript/src/main/assemblies/plugin.xml delete mode 100644 plugins/lang-javascript/src/main/resources/es-plugin.properties delete mode 100644 plugins/lang-python/src/main/assemblies/plugin.xml delete mode 100644 plugins/lang-python/src/main/resources/es-plugin.properties diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfo.java index ea55f7cce5d..927a79b6639 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfo.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfo.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.plugins.PluginInfo; import java.io.IOException; import java.util.ArrayList; @@ -75,7 +76,7 @@ public class PluginsInfo implements Streamable, ToXContent { public void readFrom(StreamInput in) throws IOException { int plugins_size = in.readInt(); for (int i = 0; i < plugins_size; i++) { - infos.add(PluginInfo.readPluginInfo(in)); + infos.add(PluginInfo.readFromStream(in)); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java index 53f43944277..046fb04028e 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -23,7 +23,7 @@ import com.carrotsearch.hppc.ObjectIntHashMap; import com.carrotsearch.hppc.cursors.ObjectIntCursor; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.PluginInfo; +import org.elasticsearch.plugins.PluginInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.StreamInput; @@ -38,7 +38,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.monitor.jvm.JvmInfo; -import org.elasticsearch.monitor.os.OsInfo; import java.io.IOException; import java.net.InetAddress; @@ -143,7 +142,7 @@ public class ClusterStatsNodes implements ToXContent, Streamable { size = in.readVInt(); plugins = new HashSet<>(size); for (; size > 0; size--) { - plugins.add(PluginInfo.readPluginInfo(in)); + plugins.add(PluginInfo.readFromStream(in)); } } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index d3e5b101aa8..36ba3bdc7a6 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -19,9 +19,6 @@ package org.elasticsearch.bootstrap; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; - import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Constants; import org.elasticsearch.ExceptionsHelper; @@ -32,7 +29,6 @@ import org.elasticsearch.common.cli.Terminal; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.CreationException; import org.elasticsearch.common.inject.spi.Message; -import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; @@ -46,20 +42,10 @@ import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.node.internal.InternalSettingsPreparer; -import java.io.IOException; -import java.lang.reflect.Method; -import java.net.URL; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.PathMatcher; -import java.util.Arrays; -import java.util.List; import java.util.Locale; import java.util.Set; import java.util.concurrent.CountDownLatch; -import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory; import static com.google.common.collect.Sets.newHashSet; import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; @@ -171,10 +157,7 @@ public class Bootstrap { } }); } - - // install any plugins into classpath - setupPlugins(environment); - + // look for jar hell JarHell.checkJarHell(); @@ -365,75 +348,5 @@ public class Bootstrap { return errorMessage.toString(); } - static final String PLUGIN_LIB_PATTERN = "glob:**.{jar,zip}"; - private static void setupPlugins(Environment environment) throws IOException { - ESLogger logger = Loggers.getLogger(Bootstrap.class); - Path pluginsDirectory = environment.pluginsFile(); - if (!isAccessibleDirectory(pluginsDirectory, logger)) { - return; - } - - // note: there's only one classloader here, but Uwe gets upset otherwise. - ClassLoader classLoader = Bootstrap.class.getClassLoader(); - Class classLoaderClass = classLoader.getClass(); - Method addURL = null; - while (!classLoaderClass.equals(Object.class)) { - try { - addURL = classLoaderClass.getDeclaredMethod("addURL", URL.class); - addURL.setAccessible(true); - break; - } catch (NoSuchMethodException e) { - // no method, try the parent - classLoaderClass = classLoaderClass.getSuperclass(); - } - } - - if (addURL == null) { - logger.debug("failed to find addURL method on classLoader [" + classLoader + "] to add methods"); - return; - } - - try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory)) { - - for (Path plugin : stream) { - // We check that subdirs are directories and readable - if (!isAccessibleDirectory(plugin, logger)) { - continue; - } - - logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath()); - - try { - // add the root - addURL.invoke(classLoader, plugin.toUri().toURL()); - // gather files to add - List libFiles = Lists.newArrayList(); - libFiles.addAll(Arrays.asList(files(plugin))); - Path libLocation = plugin.resolve("lib"); - if (Files.isDirectory(libLocation)) { - libFiles.addAll(Arrays.asList(files(libLocation))); - } - - PathMatcher matcher = PathUtils.getDefaultFileSystem().getPathMatcher(PLUGIN_LIB_PATTERN); - - // if there are jars in it, add it as well - for (Path libFile : libFiles) { - if (!matcher.matches(libFile)) { - continue; - } - addURL.invoke(classLoader, libFile.toUri().toURL()); - } - } catch (Throwable e) { - logger.warn("failed to add plugin [" + plugin + "]", e); - } - } - } - } - - private static Path[] files(Path from) throws IOException { - try (DirectoryStream stream = Files.newDirectoryStream(from)) { - return Iterators.toArray(stream.iterator(), Path.class); - } - } } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java index 4d91341296a..4c0cde1389e 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java @@ -80,7 +80,6 @@ final class Security { m.put(Pattern.compile(".*lucene-core-.*\\.jar$"), "es.security.jar.lucene.core"); m.put(Pattern.compile(".*jsr166e-.*\\.jar$"), "es.security.jar.twitter.jsr166e"); m.put(Pattern.compile(".*securemock-.*\\.jar$"), "es.security.jar.elasticsearch.securemock"); - m.put(Pattern.compile(".*bcprov-.*\\.jar$"), "es.security.jar.bouncycastle.bcprov"); SPECIAL_JARS = Collections.unmodifiableMap(m); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginInfo.java b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java similarity index 58% rename from core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginInfo.java rename to core/src/main/java/org/elasticsearch/plugins/PluginInfo.java index 205483ef0b9..6129f649460 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginInfo.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.action.admin.cluster.node.info; +package org.elasticsearch.plugins; -import org.elasticsearch.common.Strings; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -27,25 +27,34 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; public class PluginInfo implements Streamable, ToXContent { - public static final String DESCRIPTION_NOT_AVAILABLE = "No description found."; - public static final String VERSION_NOT_AVAILABLE = "NA"; + + public static final String ES_PLUGIN_PROPERTIES = "plugin-descriptor.properties"; static final class Fields { static final XContentBuilderString NAME = new XContentBuilderString("name"); static final XContentBuilderString DESCRIPTION = new XContentBuilderString("description"); static final XContentBuilderString URL = new XContentBuilderString("url"); - static final XContentBuilderString JVM = new XContentBuilderString("jvm"); static final XContentBuilderString SITE = new XContentBuilderString("site"); static final XContentBuilderString VERSION = new XContentBuilderString("version"); + static final XContentBuilderString JVM = new XContentBuilderString("jvm"); + static final XContentBuilderString CLASSNAME = new XContentBuilderString("classname"); + static final XContentBuilderString ISOLATED = new XContentBuilderString("isolated"); } private String name; private String description; private boolean site; - private boolean jvm; private String version; + + private boolean jvm; + private String classname; + private boolean isolated; public PluginInfo() { } @@ -57,18 +66,58 @@ public class PluginInfo implements Streamable, ToXContent { * @param description Its description * @param site true if it's a site plugin * @param jvm true if it's a jvm plugin - * @param version Version number is applicable (NA otherwise) + * @param version Version number */ - public PluginInfo(String name, String description, boolean site, boolean jvm, String version) { + PluginInfo(String name, String description, boolean site, String version, boolean jvm, String classname, boolean isolated) { this.name = name; this.description = description; this.site = site; this.jvm = jvm; - if (Strings.hasText(version)) { - this.version = version; - } else { - this.version = VERSION_NOT_AVAILABLE; + this.version = version; + this.classname = classname; + this.isolated = isolated; + } + + /** reads (and validates) plugin metadata descriptor file */ + public static PluginInfo readFromProperties(Path dir) throws IOException { + Path descriptor = dir.resolve(ES_PLUGIN_PROPERTIES); + Properties props = new Properties(); + try (InputStream stream = Files.newInputStream(descriptor)) { + props.load(stream); } + String name = dir.getFileName().toString(); + String description = props.getProperty("description"); + if (description == null) { + throw new IllegalArgumentException("Property [description] is missing for plugin [" + name + "]"); + } + String version = props.getProperty("version"); + if (version == null) { + throw new IllegalArgumentException("Property [version] is missing for plugin [" + name + "]"); + } + boolean jvm = Boolean.parseBoolean(props.getProperty("jvm")); + boolean site = Boolean.parseBoolean(props.getProperty("site")); + if (jvm == false && site == false) { + throw new IllegalArgumentException("Plugin [" + name + "] must be at least a jvm or site plugin"); + } + boolean isolated = true; + String classname = "NA"; + if (jvm) { + String esVersionString = props.getProperty("elasticsearch.version"); + if (esVersionString == null) { + throw new IllegalArgumentException("Property [elasticsearch.version] is missing for jvm plugin [" + name + "]"); + } + Version esVersion = Version.fromString(esVersionString); + if (esVersion.equals(Version.CURRENT) == false) { + throw new IllegalArgumentException("Elasticsearch version [" + esVersionString + "] is too old for plugin [" + name + "]"); + } + isolated = Boolean.parseBoolean(props.getProperty("isolated", "true")); + classname = props.getProperty("classname"); + if (classname == null) { + throw new IllegalArgumentException("Property [classname] is missing for jvm plugin [" + name + "]"); + } + } + + return new PluginInfo(name, description, site, version, jvm, classname, isolated); } /** @@ -98,6 +147,20 @@ public class PluginInfo implements Streamable, ToXContent { public boolean isJvm() { return jvm; } + + /** + * @return true if jvm plugin has isolated classloader + */ + public boolean isIsolated() { + return isolated; + } + + /** + * @return jvm plugin's classname + */ + public String getClassname() { + return classname; + } /** * We compute the URL for sites: "/_plugin/" + name + "/" @@ -119,7 +182,7 @@ public class PluginInfo implements Streamable, ToXContent { return version; } - public static PluginInfo readPluginInfo(StreamInput in) throws IOException { + public static PluginInfo readFromStream(StreamInput in) throws IOException { PluginInfo info = new PluginInfo(); info.readFrom(in); return info; @@ -132,6 +195,8 @@ public class PluginInfo implements Streamable, ToXContent { this.site = in.readBoolean(); this.jvm = in.readBoolean(); this.version = in.readString(); + this.classname = in.readString(); + this.isolated = in.readBoolean(); } @Override @@ -141,6 +206,8 @@ public class PluginInfo implements Streamable, ToXContent { out.writeBoolean(site); out.writeBoolean(jvm); out.writeString(version); + out.writeString(classname); + out.writeBoolean(isolated); } @Override @@ -153,6 +220,10 @@ public class PluginInfo implements Streamable, ToXContent { builder.field(Fields.URL, getUrl()); } builder.field(Fields.JVM, jvm); + if (jvm) { + builder.field(Fields.CLASSNAME, classname); + builder.field(Fields.ISOLATED, isolated); + } builder.field(Fields.SITE, site); builder.endObject(); @@ -184,6 +255,10 @@ public class PluginInfo implements Streamable, ToXContent { sb.append(", description='").append(description).append('\''); sb.append(", site=").append(site); sb.append(", jvm=").append(jvm); + if (jvm) { + sb.append(", classname=").append(classname); + sb.append(", isolated=").append(isolated); + } sb.append(", version='").append(version).append('\''); sb.append('}'); return sb.toString(); diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java index e9110da424b..158f6400bee 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -19,37 +19,43 @@ package org.elasticsearch.plugins; -import com.google.common.base.Charsets; -import com.google.common.collect.*; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; -import org.apache.lucene.util.Constants; -import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.node.info.PluginInfo; import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo; +import org.elasticsearch.bootstrap.Bootstrap; +import org.elasticsearch.bootstrap.JarHell; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.env.Environment; -import java.io.BufferedReader; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Method; import java.net.URL; -import java.nio.file.*; -import java.util.*; +import java.net.URLClassLoader; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory; @@ -57,29 +63,14 @@ import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory; * */ public class PluginsService extends AbstractComponent { - public static final String ES_PLUGIN_PROPERTIES_FILE_KEY = "plugins.properties_file"; - public static final String ES_PLUGIN_PROPERTIES = "es-plugin.properties"; - public static final String LOAD_PLUGIN_FROM_CLASSPATH = "plugins.load_classpath_plugins"; - - public static final String PLUGINS_CHECK_LUCENE_KEY = "plugins.check_lucene"; - public static final String PLUGINS_INFO_REFRESH_INTERVAL_KEY = "plugins.info_refresh_interval"; - - - private final Environment environment; /** - * We keep around a list of jvm plugins + * We keep around a list of plugins */ private final ImmutableList> plugins; + private final PluginsInfo info; private final ImmutableMap> onModuleReferences; - private final String esPluginPropertiesFile; - private final boolean loadClasspathPlugins; - - private PluginsInfo cachedPluginsInfo; - private final TimeValue refreshInterval; - private final boolean checkLucene; - private long lastRefreshNS; static class OnModuleReference { public final Class moduleClass; @@ -98,54 +89,53 @@ public class PluginsService extends AbstractComponent { */ public PluginsService(Settings settings, Environment environment) { super(settings); - this.environment = environment; - this.checkLucene = settings.getAsBoolean(PLUGINS_CHECK_LUCENE_KEY, true); - this.esPluginPropertiesFile = settings.get(ES_PLUGIN_PROPERTIES_FILE_KEY, ES_PLUGIN_PROPERTIES); - this.loadClasspathPlugins = settings.getAsBoolean(LOAD_PLUGIN_FROM_CLASSPATH, true); ImmutableList.Builder> tupleBuilder = ImmutableList.builder(); - // first we load all the default plugins from the settings + // first we load specified plugins via 'plugin.types' settings parameter. + // this is a hack for what is between unit and integration tests... String[] defaultPluginsClasses = settings.getAsArray("plugin.types"); for (String pluginClass : defaultPluginsClasses) { Plugin plugin = loadPlugin(pluginClass, settings); - PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), hasSite(plugin.name()), true, PluginInfo.VERSION_NOT_AVAILABLE); + PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), false, "NA", true, pluginClass, false); if (logger.isTraceEnabled()) { logger.trace("plugin loaded from settings [{}]", pluginInfo); } tupleBuilder.add(new Tuple<>(pluginInfo, plugin)); } - // now, find all the ones that are in the classpath - if (loadClasspathPlugins) { - tupleBuilder.addAll(loadPluginsFromClasspath(settings)); + // now, find all the ones that are in plugins/ + try { + List bundles = getPluginBundles(environment); + tupleBuilder.addAll(loadBundles(bundles)); + } catch (IOException ex) { + throw new IllegalStateException("Can't load plugins into classloader", ex); + } + + plugins = tupleBuilder.build(); + info = new PluginsInfo(); + for (Tuple tuple : plugins) { + info.add(tuple.v1()); } - this.plugins = tupleBuilder.build(); // We need to build a List of jvm and site plugins for checking mandatory plugins - Map jvmPlugins = Maps.newHashMap(); - List sitePlugins = Lists.newArrayList(); + Map jvmPlugins = new HashMap<>(); + List sitePlugins = new ArrayList<>(); - for (Tuple tuple : this.plugins) { - jvmPlugins.put(tuple.v2().name(), tuple.v2()); - if (tuple.v1().isSite()) { - sitePlugins.add(tuple.v1().getName()); + for (Tuple tuple : plugins) { + PluginInfo info = tuple.v1(); + if (info.isJvm()) { + jvmPlugins.put(tuple.v2().name(), tuple.v2()); } - } - try { - // we load site plugins - ImmutableList> tuples = loadSitePlugins(); - for (Tuple tuple : tuples) { - sitePlugins.add(tuple.v1().getName()); + if (info.isSite()) { + sitePlugins.add(info.getName()); } - } catch (IOException ex) { - throw new IllegalStateException("Can't load site plugins", ex); } // Checking expected plugins String[] mandatoryPlugins = settings.getAsArray("plugin.mandatory", null); if (mandatoryPlugins != null) { - Set missingPlugins = Sets.newHashSet(); + Set missingPlugins = new HashSet<>(); for (String mandatoryPlugin : mandatoryPlugins) { if (!jvmPlugins.containsKey(mandatoryPlugin) && !sitePlugins.contains(mandatoryPlugin) && !missingPlugins.contains(mandatoryPlugin)) { missingPlugins.add(mandatoryPlugin); @@ -160,8 +150,8 @@ public class PluginsService extends AbstractComponent { MapBuilder> onModuleReferences = MapBuilder.newMapBuilder(); for (Plugin plugin : jvmPlugins.values()) { - List list = Lists.newArrayList(); - for (Method method : plugin.getClass().getDeclaredMethods()) { + List list = new ArrayList<>(); + for (Method method : plugin.getClass().getMethods()) { if (!method.getName().equals("onModule")) { continue; } @@ -174,7 +164,6 @@ public class PluginsService extends AbstractComponent { logger.warn("Plugin: {} implementing onModule by the type is not of Module type {}", plugin.name(), moduleClass); continue; } - method.setAccessible(true); list.add(new OnModuleReference(moduleClass, method)); } if (!list.isEmpty()) { @@ -182,8 +171,6 @@ public class PluginsService extends AbstractComponent { } } this.onModuleReferences = onModuleReferences.immutableMap(); - - this.refreshInterval = settings.getAsTime(PLUGINS_INFO_REFRESH_INTERVAL_KEY, TimeValue.timeValueSeconds(10)); } public ImmutableList> plugins() { @@ -225,7 +212,7 @@ public class PluginsService extends AbstractComponent { } public Collection> modules() { - List> modules = Lists.newArrayList(); + List> modules = new ArrayList<>(); for (Tuple plugin : plugins) { modules.addAll(plugin.v2().modules()); } @@ -233,7 +220,7 @@ public class PluginsService extends AbstractComponent { } public Collection modules(Settings settings) { - List modules = Lists.newArrayList(); + List modules = new ArrayList<>(); for (Tuple plugin : plugins) { modules.addAll(plugin.v2().modules(settings)); } @@ -241,7 +228,7 @@ public class PluginsService extends AbstractComponent { } public Collection> services() { - List> services = Lists.newArrayList(); + List> services = new ArrayList<>(); for (Tuple plugin : plugins) { services.addAll(plugin.v2().services()); } @@ -249,7 +236,7 @@ public class PluginsService extends AbstractComponent { } public Collection> indexModules() { - List> modules = Lists.newArrayList(); + List> modules = new ArrayList<>(); for (Tuple plugin : plugins) { modules.addAll(plugin.v2().indexModules()); } @@ -257,7 +244,7 @@ public class PluginsService extends AbstractComponent { } public Collection indexModules(Settings settings) { - List modules = Lists.newArrayList(); + List modules = new ArrayList<>(); for (Tuple plugin : plugins) { modules.addAll(plugin.v2().indexModules(settings)); } @@ -265,7 +252,7 @@ public class PluginsService extends AbstractComponent { } public Collection> indexServices() { - List> services = Lists.newArrayList(); + List> services = new ArrayList<>(); for (Tuple plugin : plugins) { services.addAll(plugin.v2().indexServices()); } @@ -273,7 +260,7 @@ public class PluginsService extends AbstractComponent { } public Collection> shardModules() { - List> modules = Lists.newArrayList(); + List> modules = new ArrayList<>(); for (Tuple plugin : plugins) { modules.addAll(plugin.v2().shardModules()); } @@ -281,7 +268,7 @@ public class PluginsService extends AbstractComponent { } public Collection shardModules(Settings settings) { - List modules = Lists.newArrayList(); + List modules = new ArrayList<>(); for (Tuple plugin : plugins) { modules.addAll(plugin.v2().shardModules(settings)); } @@ -289,7 +276,7 @@ public class PluginsService extends AbstractComponent { } public Collection> shardServices() { - List> services = Lists.newArrayList(); + List> services = new ArrayList<>(); for (Tuple plugin : plugins) { services.addAll(plugin.v2().shardServices()); } @@ -298,249 +285,125 @@ public class PluginsService extends AbstractComponent { /** * Get information about plugins (jvm and site plugins). - * Information are cached for 10 seconds by default. Modify `plugins.info_refresh_interval` property if needed. - * Setting `plugins.info_refresh_interval` to `-1` will cause infinite caching. - * Setting `plugins.info_refresh_interval` to `0` will disable caching. - * @return List of plugins information */ - synchronized public PluginsInfo info() { - if (refreshInterval.millis() != 0) { - if (cachedPluginsInfo != null && - (refreshInterval.millis() < 0 || (System.nanoTime() - lastRefreshNS) < refreshInterval.nanos())) { - if (logger.isTraceEnabled()) { - logger.trace("using cache to retrieve plugins info"); - } - return cachedPluginsInfo; - } - lastRefreshNS = System.nanoTime(); - } - - if (logger.isTraceEnabled()) { - logger.trace("starting to fetch info on plugins"); - } - cachedPluginsInfo = new PluginsInfo(); - - // We first add all JvmPlugins - for (Tuple plugin : this.plugins) { - if (logger.isTraceEnabled()) { - logger.trace("adding jvm plugin [{}]", plugin.v1()); - } - cachedPluginsInfo.add(plugin.v1()); - } - - try { - // We reload site plugins (in case of some changes) - for (Tuple plugin : loadSitePlugins()) { - if (logger.isTraceEnabled()) { - logger.trace("adding site plugin [{}]", plugin.v1()); - } - cachedPluginsInfo.add(plugin.v1()); - } - } catch (IOException ex) { - logger.warn("can load site plugins info", ex); - } - - return cachedPluginsInfo; + public PluginsInfo info() { + return info; + } + + // a "bundle" is a group of plugins in a single classloader + // really should be 1-1, but we are not so fortunate + static class Bundle { + List plugins = new ArrayList<>(); + List urls = new ArrayList<>(); } + static List getPluginBundles(Environment environment) throws IOException { + ESLogger logger = Loggers.getLogger(Bootstrap.class); + Path pluginsDirectory = environment.pluginsFile(); + if (!isAccessibleDirectory(pluginsDirectory, logger)) { + return Collections.emptyList(); + } + + List bundles = new ArrayList<>(); + // a special purgatory for plugins that directly depend on each other + bundles.add(new Bundle()); - private List> loadPluginsFromClasspath(Settings settings) { - ImmutableList.Builder> plugins = ImmutableList.builder(); - - // Trying JVM plugins: looking for es-plugin.properties files - try { - Enumeration pluginUrls = settings.getClassLoader().getResources(esPluginPropertiesFile); - - // use a set for uniqueness as some classloaders such as groovy's can return the same URL multiple times and - // these plugins should only be loaded once - HashSet uniqueUrls = new HashSet<>(Collections.list(pluginUrls)); - for (URL pluginUrl : uniqueUrls) { - Properties pluginProps = new Properties(); - InputStream is = null; + try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory)) { + for (Path plugin : stream) { try { - is = pluginUrl.openStream(); - pluginProps.load(is); - String pluginClassName = pluginProps.getProperty("plugin"); - String pluginVersion = pluginProps.getProperty("version", PluginInfo.VERSION_NOT_AVAILABLE); - Plugin plugin = loadPlugin(pluginClassName, settings); - - // Is it a site plugin as well? Does it have also an embedded _site structure - Path siteFile = environment.pluginsFile().resolve(plugin.name()).resolve("_site"); - boolean isSite = isAccessibleDirectory(siteFile, logger); - if (logger.isTraceEnabled()) { - logger.trace("found a jvm plugin [{}], [{}]{}", - plugin.name(), plugin.description(), isSite ? ": with _site structure" : ""); + logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath()); + PluginInfo info = PluginInfo.readFromProperties(plugin); + List urls = new ArrayList<>(); + if (info.isJvm()) { + // a jvm plugin: gather urls for jar files + try (DirectoryStream jarStream = Files.newDirectoryStream(plugin, "*.jar")) { + for (Path jar : jarStream) { + urls.add(jar.toUri().toURL()); + } + } } - - PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), isSite, true, pluginVersion); - - plugins.add(new Tuple<>(pluginInfo, plugin)); + final Bundle bundle; + if (info.isJvm() && info.isIsolated() == false) { + bundle = bundles.get(0); // purgatory + } else { + bundle = new Bundle(); + bundles.add(bundle); + } + bundle.plugins.add(info); + bundle.urls.addAll(urls); } catch (Throwable e) { - logger.warn("failed to load plugin from [" + pluginUrl + "]", e); - } finally { - IOUtils.closeWhileHandlingException(is); + logger.warn("failed to add plugin [" + plugin + "]", e); + } + } + } + + return bundles; + } + + private List> loadBundles(List bundles) { + ImmutableList.Builder> plugins = ImmutableList.builder(); + + for (Bundle bundle : bundles) { + // jar-hell check the bundle against the parent classloader + // pluginmanager does it, but we do it again, in case lusers mess with jar files manually + try { + final List jars = new ArrayList<>(); + ClassLoader parentLoader = settings.getClassLoader(); + if (parentLoader instanceof URLClassLoader) { + for (URL url : ((URLClassLoader) parentLoader).getURLs()) { + jars.add(url); + } + } + jars.addAll(bundle.urls); + JarHell.checkJarHell(jars.toArray(new URL[0])); + } catch (Exception e) { + logger.warn("failed to load bundle {} due to jar hell", bundle.urls); + } + + // create a child to load the plugins in this bundle + ClassLoader loader = URLClassLoader.newInstance(bundle.urls.toArray(new URL[0]), settings.getClassLoader()); + Settings settings = Settings.builder() + .put(this.settings) + .classLoader(loader) + .build(); + + for (PluginInfo pluginInfo : bundle.plugins) { + try { + final Plugin plugin; + if (pluginInfo.isJvm()) { + plugin = loadPlugin(pluginInfo.getClassname(), settings); + } else { + plugin = null; + } + plugins.add(new Tuple<>(pluginInfo, plugin)); + } catch (Throwable e) { + logger.warn("failed to load plugin from [" + bundle.urls + "]", e); } } - } catch (IOException e) { - logger.warn("failed to find jvm plugins from classpath", e); } return plugins.build(); } - private ImmutableList> loadSitePlugins() throws IOException { - ImmutableList.Builder> sitePlugins = ImmutableList.builder(); - List loadedJvmPlugins = new ArrayList<>(); - - // Already known jvm plugins are ignored - for(Tuple tuple : plugins) { - if (tuple.v1().isSite()) { - loadedJvmPlugins.add(tuple.v1().getName()); - } - } - - // Let's try to find all _site plugins we did not already found - Path pluginsFile = environment.pluginsFile(); - - if (FileSystemUtils.isAccessibleDirectory(pluginsFile, logger) == false) { - return sitePlugins.build(); - } - - try (DirectoryStream stream = Files.newDirectoryStream(pluginsFile)) { - for (Path pluginFile : stream) { - if (!loadedJvmPlugins.contains(pluginFile.getFileName().toString())) { - Path sitePluginDir = pluginFile.resolve("_site"); - if (isAccessibleDirectory(sitePluginDir, logger)) { - // We have a _site plugin. Let's try to get more information on it - String name = pluginFile.getFileName().toString(); - String version = PluginInfo.VERSION_NOT_AVAILABLE; - String description = PluginInfo.DESCRIPTION_NOT_AVAILABLE; - - // We check if es-plugin.properties exists in plugin/_site dir - final Path pluginPropFile = sitePluginDir.resolve(esPluginPropertiesFile); - if (Files.exists(pluginPropFile)) { - - final Properties pluginProps = new Properties(); - try (final BufferedReader reader = Files.newBufferedReader(pluginPropFile, Charsets.UTF_8)) { - pluginProps.load(reader); - description = pluginProps.getProperty("description", PluginInfo.DESCRIPTION_NOT_AVAILABLE); - version = pluginProps.getProperty("version", PluginInfo.VERSION_NOT_AVAILABLE); - } catch (Exception e) { - // Can not load properties for this site plugin. Ignoring. - logger.debug("can not load {} file.", e, esPluginPropertiesFile); - } - } - - if (logger.isTraceEnabled()) { - logger.trace("found a site plugin name [{}], version [{}], description [{}]", - name, version, description); - } - sitePlugins.add(new Tuple(new PluginInfo(name, description, true, false, version), null)); - } - } - } - } - return sitePlugins.build(); - } - - /** - * @param name plugin name - * @return if this jvm plugin has also a _site structure - */ - private boolean hasSite(String name) { - // Let's try to find all _site plugins we did not already found - Path pluginsFile = environment.pluginsFile(); - - if (!Files.isDirectory(pluginsFile)) { - return false; - } - - Path sitePluginDir = pluginsFile.resolve(name).resolve("_site"); - return isAccessibleDirectory(sitePluginDir, logger); - } - private Plugin loadPlugin(String className, Settings settings) { try { - Class pluginClass = (Class) settings.getClassLoader().loadClass(className); - Plugin plugin; + Class pluginClass = settings.getClassLoader().loadClass(className).asSubclass(Plugin.class); - if (!checkLucene || checkLuceneCompatibility(pluginClass, settings, logger, esPluginPropertiesFile)) { + try { + return pluginClass.getConstructor(Settings.class).newInstance(settings); + } catch (NoSuchMethodException e) { try { - plugin = pluginClass.getConstructor(Settings.class).newInstance(settings); - } catch (NoSuchMethodException e) { - try { - plugin = pluginClass.getConstructor().newInstance(); - } catch (NoSuchMethodException e1) { - throw new ElasticsearchException("No constructor for [" + pluginClass + "]. A plugin class must " + - "have either an empty default constructor or a single argument constructor accepting a " + - "Settings instance"); - } + return pluginClass.getConstructor().newInstance(); + } catch (NoSuchMethodException e1) { + throw new ElasticsearchException("No constructor for [" + pluginClass + "]. A plugin class must " + + "have either an empty default constructor or a single argument constructor accepting a " + + "Settings instance"); } - } else { - throw new ElasticsearchException("Plugin is incompatible with the current node"); } - - return plugin; - } catch (Throwable e) { throw new ElasticsearchException("Failed to load plugin class [" + className + "]", e); } } - - /** - *

Check that a plugin is Lucene compatible with the current running node using `lucene` property - * in `es-plugin.properties` file.

- *

If plugin does not provide `lucene` property, we consider that the plugin is compatible.

- *

If plugin provides `lucene` property, we try to load related Enum org.apache.lucene.util.Version. If this - * fails, it means that the node is too "old" comparing to the Lucene version the plugin was built for.

- *

We compare then two first digits of current node lucene version against two first digits of plugin Lucene - * version. If not equal, it means that the plugin is too "old" for the current node.

- * - * @param pluginClass Plugin class we are checking - * @return true if the plugin is Lucene compatible - */ - public static boolean checkLuceneCompatibility(Class pluginClass, Settings settings, ESLogger logger, String propertiesFile) { - String luceneVersion = null; - try { - // We try to read the es-plugin.properties file - // But as we can have several plugins in the same classloader, - // we have to find the right es-plugin.properties file - Enumeration pluginUrls = settings.getClassLoader().getResources(propertiesFile); - - while (pluginUrls.hasMoreElements()) { - URL pluginUrl = pluginUrls.nextElement(); - try (InputStream is = pluginUrl.openStream()) { - Properties pluginProps = new Properties(); - pluginProps.load(is); - String plugin = pluginProps.getProperty("plugin"); - // If we don't have the expected plugin, let's continue to the next one - if (pluginClass.getName().equals(plugin)) { - luceneVersion = pluginProps.getProperty("lucene"); - break; - } - logger.debug("skipping [{}]", pluginUrl); - } - } - - if (luceneVersion != null) { - // Should fail if the running node is too old! - org.apache.lucene.util.Version luceneExpectedVersion = Lucene.parseVersionLenient(luceneVersion, null); - if (Version.CURRENT.luceneVersion.equals(luceneExpectedVersion)) { - logger.debug("starting analysis plugin for Lucene [{}].", luceneExpectedVersion); - return true; - } - } else { - logger.debug("lucene property is not set in plugin {} file. Skipping test.", propertiesFile); - return true; - } - } catch (Throwable t) { - // We don't have the expected version... Let's fail after. - logger.debug("exception raised while checking plugin Lucene version.", t); - } - logger.error("cannot start plugin due to incorrect Lucene version: plugin [{}], node [{}].", - luceneVersion, Constants.LUCENE_MAIN_VERSION); - return false; - } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java index 3d0dda320e1..d9170a75c41 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java @@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.cat; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.node.info.PluginInfo; +import org.elasticsearch.plugins.PluginInfo; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.client.Client; diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index 157c3cffeb0..7d77900bb71 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -47,11 +47,6 @@ grant codeBase "${es.security.jar.elasticsearch.securemock}" { permission java.lang.RuntimePermission "reflectionFactoryAccess"; }; -grant codeBase "${es.security.jar.bouncycastle.bcprov}" { - // needed to allow installation of bouncycastle crypto provider - permission java.security.SecurityPermission "putProviderProperty.BC"; -}; - //// Everything else: grant { @@ -123,4 +118,7 @@ grant { // needed to install SSLFactories, advanced SSL configuration, etc. permission java.lang.RuntimePermission "setFactory"; + + // needed to allow installation of bouncycastle crypto provider + permission java.security.SecurityPermission "putProviderProperty.BC"; }; diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfoTest.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfoTest.java deleted file mode 100644 index feb7f5a8938..00000000000 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/info/PluginsInfoTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.cluster.node.info; - -import com.google.common.base.Function; -import com.google.common.collect.Lists; -import org.elasticsearch.test.ElasticsearchTestCase; -import org.junit.Test; - -import java.util.List; - -import static org.hamcrest.Matchers.contains; - -public class PluginsInfoTest extends ElasticsearchTestCase { - - @Test - public void testPluginListSorted() { - PluginsInfo pluginsInfo = new PluginsInfo(5); - pluginsInfo.add(new PluginInfo("c", "foo", true, true, "dummy")); - pluginsInfo.add(new PluginInfo("b", "foo", true, true, "dummy")); - pluginsInfo.add(new PluginInfo("e", "foo", true, true, "dummy")); - pluginsInfo.add(new PluginInfo("a", "foo", true, true, "dummy")); - pluginsInfo.add(new PluginInfo("d", "foo", true, true, "dummy")); - - final List infos = pluginsInfo.getInfos(); - List names = Lists.transform(infos, new Function() { - @Override - public String apply(PluginInfo input) { - return input.getName(); - } - }); - assertThat(names, contains("a", "b", "c", "d", "e")); - } -} diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java deleted file mode 100644 index c07995e8d35..00000000000 --- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.bootstrap; - -import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.test.ElasticsearchTestCase; -import org.junit.Test; - -import java.nio.file.Path; -import java.nio.file.PathMatcher; - -public class BootstrapTests extends ElasticsearchTestCase { - - @Test - public void testHasLibExtension() { - PathMatcher matcher = PathUtils.getDefaultFileSystem().getPathMatcher(Bootstrap.PLUGIN_LIB_PATTERN); - - Path p = PathUtils.get("path", "to", "plugin.jar"); - assertTrue(matcher.matches(p)); - - p = PathUtils.get("path", "to", "plugin.zip"); - assertTrue(matcher.matches(p)); - - p = PathUtils.get("path", "to", "plugin.tar.gz"); - assertFalse(matcher.matches(p)); - - p = PathUtils.get("path", "to", "plugin"); - assertFalse(matcher.matches(p)); - } -} diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryTests.java index 8a5111c6748..6557b810293 100644 --- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryTests.java +++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryTests.java @@ -62,7 +62,6 @@ public class TransportClientRetryTests extends ElasticsearchIntegrationTest { Settings.Builder builder = settingsBuilder().put("client.transport.nodes_sampler_interval", "1s") .put("name", "transport_client_retry_test") .put("node.mode", InternalTestCluster.nodeMode()) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false) .put(ClusterName.SETTING, internalCluster().getClusterName()) .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true) .put("path.home", createTempDir()); diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoTests.java index 7b94b5ef117..68f0281b621 100644 --- a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoTests.java +++ b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoTests.java @@ -23,7 +23,6 @@ import com.google.common.collect.Lists; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.node.info.PluginInfo; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.nodesinfo.plugin.dummy1.TestPlugin; @@ -94,66 +93,4 @@ public class SimpleNodesInfoTests extends PluginTestCase { assertThat(response.getNodes().length, is(1)); assertThat(response.getNodesMap().get(server2NodeId), notNullValue()); } - - /** - * Use case is to start 4 nodes: - *
    - *
  • 1 : no plugin
    • - *
  • 2 : one site plugin (with a es-plugin.properties file)
    • - *
  • 3 : one java plugin
    • - *
  • 4 : one site plugin and 2 java plugins (included the previous one)
    • - *
- * We test here that NodeInfo API with plugin option give us the right results. - * @throws URISyntaxException - */ - @Test - public void testNodeInfoPlugin() throws URISyntaxException { - // We start four nodes - // The first has no plugin - String server1NodeId = startNodeWithPlugins(1); - // The second has one site plugin with a es-plugin.properties file (description and version) - String server2NodeId = startNodeWithPlugins(2); - // The third has one java plugin - String server3NodeId = startNodeWithPlugins(3,TestPlugin.class.getName()); - // The fourth has one java plugin and one site plugin - String server4NodeId = startNodeWithPlugins(4,TestNoVersionPlugin.class.getName()); - - ClusterHealthResponse clusterHealth = client().admin().cluster().health(clusterHealthRequest().waitForNodes("4")).actionGet(); - logger.info("--> done cluster_health, status " + clusterHealth.getStatus()); - - NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().clear().setPlugins(true).execute().actionGet(); - logger.info("--> full json answer, status " + response.toString()); - - ElasticsearchAssertions.assertNodeContainsPlugins(response, server1NodeId, - Collections.EMPTY_LIST, Collections.EMPTY_LIST, Collections.EMPTY_LIST, // No JVM Plugin - Collections.EMPTY_LIST, Collections.EMPTY_LIST, Collections.EMPTY_LIST);// No Site Plugin - - ElasticsearchAssertions.assertNodeContainsPlugins(response, server2NodeId, - Collections.EMPTY_LIST, Collections.EMPTY_LIST, Collections.EMPTY_LIST, // No JVM Plugin - Lists.newArrayList(Fields.SITE_PLUGIN), // Site Plugin - Lists.newArrayList(Fields.SITE_PLUGIN_DESCRIPTION), - Lists.newArrayList(Fields.SITE_PLUGIN_VERSION)); - - ElasticsearchAssertions.assertNodeContainsPlugins(response, server3NodeId, - Lists.newArrayList(TestPlugin.Fields.NAME), // JVM Plugin - Lists.newArrayList(TestPlugin.Fields.DESCRIPTION), - Lists.newArrayList(PluginInfo.VERSION_NOT_AVAILABLE), - Collections.EMPTY_LIST, Collections.EMPTY_LIST, Collections.EMPTY_LIST);// No site Plugin - - ElasticsearchAssertions.assertNodeContainsPlugins(response, server4NodeId, - Lists.newArrayList(TestNoVersionPlugin.Fields.NAME), // JVM Plugin - Lists.newArrayList(TestNoVersionPlugin.Fields.DESCRIPTION), - Lists.newArrayList(PluginInfo.VERSION_NOT_AVAILABLE), - Lists.newArrayList(Fields.SITE_PLUGIN, TestNoVersionPlugin.Fields.NAME),// Site Plugin - Lists.newArrayList(PluginInfo.DESCRIPTION_NOT_AVAILABLE), - Lists.newArrayList(PluginInfo.VERSION_NOT_AVAILABLE)); - } - - public String startNodeWithPlugins(int nodeId, String ... pluginClassNames) throws URISyntaxException { - return startNodeWithPlugins(Settings.EMPTY, "/org/elasticsearch/nodesinfo/node" + Integer.toString(nodeId) + "/", pluginClassNames); - } - - - - } diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java new file mode 100644 index 00000000000..9f365ecdc67 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java @@ -0,0 +1,197 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugins; + +import com.google.common.base.Function; +import com.google.common.collect.Lists; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo; +import org.elasticsearch.plugins.PluginInfo; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Properties; + +import static org.hamcrest.Matchers.contains; + +public class PluginInfoTests extends ElasticsearchTestCase { + + void writeProperties(Path pluginDir, String... stringProps) throws IOException { + assert stringProps.length % 2 == 0; + Files.createDirectories(pluginDir); + Path propertiesFile = pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES); + Properties properties = new Properties(); + for (int i = 0; i < stringProps.length; i += 2) { + properties.put(stringProps[i], stringProps[i + 1]); + } + try (OutputStream out = Files.newOutputStream(propertiesFile)) { + properties.store(out, ""); + } + } + + public void testReadFromProperties() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + writeProperties(pluginDir, + "description", "fake desc", + "version", "1.0", + "elasticsearch.version", Version.CURRENT.toString(), + "jvm", "true", + "classname", "FakePlugin"); + PluginInfo info = PluginInfo.readFromProperties(pluginDir); + assertEquals("fake-plugin", info.getName()); + assertEquals("fake desc", info.getDescription()); + assertEquals("1.0", info.getVersion()); + assertEquals("FakePlugin", info.getClassname()); + assertTrue(info.isJvm()); + assertTrue(info.isIsolated()); + assertFalse(info.isSite()); + assertNull(info.getUrl()); + } + + public void testReadFromPropertiesDescriptionMissing() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + writeProperties(pluginDir); + try { + PluginInfo.readFromProperties(pluginDir); + fail("expected missing description exception"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("[description] is missing")); + } + } + + public void testReadFromPropertiesVersionMissing() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + writeProperties(pluginDir, "description", "fake desc"); + try { + PluginInfo.readFromProperties(pluginDir); + fail("expected missing version exception"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("[version] is missing")); + } + } + + public void testReadFromPropertiesJvmAndSiteMissing() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + writeProperties(pluginDir, + "description", "fake desc", + "version", "1.0"); + try { + PluginInfo.readFromProperties(pluginDir); + fail("expected jvm or site exception"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("must be at least a jvm or site plugin")); + } + } + + public void testReadFromPropertiesElasticsearchVersionMissing() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + writeProperties(pluginDir, + "description", "fake desc", + "version", "1.0", + "jvm", "true"); + try { + PluginInfo.readFromProperties(pluginDir); + fail("expected missing elasticsearch version exception"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("[elasticsearch.version] is missing")); + } + } + + public void testReadFromPropertiesBogusElasticsearchVersion() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + writeProperties(pluginDir, + "description", "fake desc", + "version", "1.0", + "jvm", "true", + "elasticsearch.version", "bogus"); + try { + PluginInfo.readFromProperties(pluginDir); + fail("expected bogus elasticsearch version exception"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("version needs to contain major, minor and revision")); + } + } + + public void testReadFromPropertiesOldElasticsearchVersion() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + writeProperties(pluginDir, + "description", "fake desc", + "version", "1.0", + "jvm", "true", + "elasticsearch.version", Version.V_1_7_0.toString()); + try { + PluginInfo.readFromProperties(pluginDir); + fail("expected old elasticsearch version exception"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Elasticsearch version [1.7.0] is too old")); + } + } + + public void testReadFromPropertiesJvmMissingClassname() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + writeProperties(pluginDir, + "description", "fake desc", + "version", "1.0", + "elasticsearch.version", Version.CURRENT.toString(), + "jvm", "true"); + try { + PluginInfo.readFromProperties(pluginDir); + fail("expected old elasticsearch version exception"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Property [classname] is missing")); + } + } + + public void testReadFromPropertiesSitePlugin() throws Exception { + Path pluginDir = createTempDir().resolve("fake-plugin"); + writeProperties(pluginDir, + "description", "fake desc", + "version", "1.0", + "elasticsearch.version", Version.CURRENT.toString(), + "site", "true"); + PluginInfo info = PluginInfo.readFromProperties(pluginDir); + assertTrue(info.isSite()); + assertFalse(info.isJvm()); + assertEquals("NA", info.getClassname()); + } + + public void testPluginListSorted() { + PluginsInfo pluginsInfo = new PluginsInfo(5); + pluginsInfo.add(new PluginInfo("c", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.add(new PluginInfo("b", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.add(new PluginInfo("e", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.add(new PluginInfo("a", "foo", true, "dummy", true, "dummyclass", true)); + pluginsInfo.add(new PluginInfo("d", "foo", true, "dummy", true, "dummyclass", true)); + + final List infos = pluginsInfo.getInfos(); + List names = Lists.transform(infos, new Function() { + @Override + public String apply(PluginInfo input) { + return input.getName(); + } + }); + assertThat(names, contains("a", "b", "c", "d", "e")); + } +} diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java index 2641db65d0f..98ba4b6fbc1 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java @@ -18,19 +18,14 @@ */ package org.elasticsearch.plugins; -import com.google.common.base.Predicate; import org.apache.http.impl.client.HttpClients; import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.node.info.PluginInfo; import org.elasticsearch.common.cli.CliTool; import org.elasticsearch.common.cli.CliToolTestCase.CaptureOutputTerminal; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.node.internal.InternalSettingsPreparer; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.junit.annotations.Network; @@ -48,7 +43,6 @@ import java.nio.file.attribute.PosixFileAttributeView; import java.nio.file.attribute.PosixFileAttributes; import java.nio.file.attribute.PosixFilePermission; import java.util.Locale; -import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.cli.CliTool.ExitStatus.USAGE; import static org.elasticsearch.common.cli.CliToolTestCase.args; @@ -90,23 +84,10 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest { @Test public void testThatPluginNameMustBeSupplied() throws IOException { - String pluginUrl = getPluginUrlForResource("plugin_single_folder.zip"); + String pluginUrl = getPluginUrlForResource("plugin_with_bin_and_config.zip"); assertStatus("install --url " + pluginUrl, USAGE); } - @Test - public void testLocalPluginInstallSingleFolder() throws Exception { - //When we have only a folder in top-level (no files either) we remove that folder while extracting - String pluginName = "plugin-test"; - String pluginUrl = getPluginUrlForResource("plugin_single_folder.zip"); - String installCommand = String.format(Locale.ROOT, "install %s --url %s", pluginName, pluginUrl); - assertStatusOk(installCommand); - - internalCluster().startNode(initialSettings.v1()); - assertPluginLoaded(pluginName); - assertPluginAvailable(pluginName); - } - @Test public void testLocalPluginInstallWithBinAndConfig() throws Exception { String pluginName = "plugin-test"; @@ -215,43 +196,6 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest { assertDirectoryExists(pluginBinDir); } - @Test - public void testLocalPluginInstallSiteFolder() throws Exception { - //When we have only a folder in top-level (no files either) but it's called _site, we make it work - //we can either remove the folder while extracting and then re-add it manually or just leave it as it is - String pluginName = "plugin-test"; - assertStatusOk(String.format(Locale.ROOT, "install %s --url %s --verbose", pluginName, getPluginUrlForResource("plugin_folder_site.zip"))); - - internalCluster().startNode(initialSettings.v1()); - - assertPluginLoaded(pluginName); - assertPluginAvailable(pluginName); - } - - @Test - public void testLocalPluginWithoutFolders() throws Exception { - //When we don't have folders at all in the top-level, but only files, we don't modify anything - String pluginName = "plugin-test"; - assertStatusOk(String.format(Locale.ROOT, "install %s --url %s --verbose", pluginName, getPluginUrlForResource("plugin_without_folders.zip"))); - - internalCluster().startNode(initialSettings.v1()); - - assertPluginLoaded(pluginName); - assertPluginAvailable(pluginName); - } - - @Test - public void testLocalPluginFolderAndFile() throws Exception { - //When we have a single top-level folder but also files in the top-level, we don't modify anything - String pluginName = "plugin-test"; - assertStatusOk(String.format(Locale.ROOT, "install %s --url %s --verbose", pluginName, getPluginUrlForResource("plugin_folder_file.zip"))); - - internalCluster().startNode(initialSettings.v1()); - - assertPluginLoaded(pluginName); - assertPluginAvailable(pluginName); - } - @Test public void testSitePluginWithSourceDoesNotInstall() throws Exception { String pluginName = "plugin-with-source"; @@ -261,54 +205,6 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest { assertThat(terminal.getTerminalOutput(), hasItem(containsString("Plugin installation assumed to be site plugin, but contains source code, aborting installation"))); } - private void assertPluginLoaded(String pluginName) { - NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().clear().setPlugins(true).get(); - assertThat(nodesInfoResponse.getNodes().length, equalTo(1)); - assertThat(nodesInfoResponse.getNodes()[0].getPlugins().getInfos(), notNullValue()); - assertThat(nodesInfoResponse.getNodes()[0].getPlugins().getInfos().size(), not(0)); - - boolean pluginFound = false; - - for (PluginInfo pluginInfo : nodesInfoResponse.getNodes()[0].getPlugins().getInfos()) { - if (pluginInfo.getName().equals(pluginName)) { - pluginFound = true; - break; - } - } - - assertThat(pluginFound, is(true)); - } - - private void assertPluginAvailable(String pluginName) throws InterruptedException, IOException { - final HttpRequestBuilder httpRequestBuilder = httpClient(); - - //checking that the http connector is working properly - // We will try it for some seconds as it could happen that the REST interface is not yet fully started - assertThat(awaitBusy(new Predicate() { - @Override - public boolean apply(Object obj) { - try { - HttpResponse response = httpRequestBuilder.method("GET").path("/").execute(); - if (response.getStatusCode() != RestStatus.OK.getStatus()) { - // We want to trace what's going on here before failing the test - logger.info("--> error caught [{}], headers [{}]", response.getStatusCode(), response.getHeaders()); - logger.info("--> cluster state [{}]", internalCluster().clusterService().state()); - return false; - } - return true; - } catch (IOException e) { - throw new ElasticsearchException("HTTP problem", e); - } - } - }, 5, TimeUnit.SECONDS), equalTo(true)); - - - //checking now that the plugin is available - HttpResponse response = httpClient().method("GET").path("/_plugin/" + pluginName + "/").execute(); - assertThat(response, notNullValue()); - assertThat(response.getReasonPhrase(), response.getStatusCode(), equalTo(RestStatus.OK.getStatus())); - } - @Test public void testListInstalledEmpty() throws IOException { assertStatusOk("list"); @@ -394,7 +290,7 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest { * It should find it on github */ @Test - @Network + @Network @AwaitsFix(bugUrl = "needs to be adapted to 2.0") public void testInstallPluginWithGithub() throws IOException { assumeTrue("github.com is accessible", isDownloadServiceWorking("github.com", 443, "/")); singlePluginInstallAndRemove("elasticsearch/kibana", "kibana", null); diff --git a/core/src/test/java/org/elasticsearch/plugins/lucene/current/CurrentLucenePlugin.java b/core/src/test/java/org/elasticsearch/plugins/lucene/current/CurrentLucenePlugin.java deleted file mode 100644 index 0b2aeb1f4bb..00000000000 --- a/core/src/test/java/org/elasticsearch/plugins/lucene/current/CurrentLucenePlugin.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.plugins.lucene.current; - -import org.elasticsearch.plugins.AbstractPlugin; - -public class CurrentLucenePlugin extends AbstractPlugin { - /** - * The name of the plugin. - */ - @Override - public String name() { - return "current-lucene"; - } - - /** - * The description of the plugin. - */ - @Override - public String description() { - return "current"; - } -} diff --git a/core/src/test/java/org/elasticsearch/plugins/lucene/current/es-plugin-test.properties b/core/src/test/java/org/elasticsearch/plugins/lucene/current/es-plugin-test.properties deleted file mode 100644 index eca345b0a28..00000000000 --- a/core/src/test/java/org/elasticsearch/plugins/lucene/current/es-plugin-test.properties +++ /dev/null @@ -1,21 +0,0 @@ -################################################################ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -################################################################ -plugin=org.elasticsearch.plugins.lucene.current.CurrentLucenePlugin -version=2.0.0 -lucene=${lucene.version} diff --git a/core/src/test/java/org/elasticsearch/plugins/lucene/newer/NewerLucenePlugin.java b/core/src/test/java/org/elasticsearch/plugins/lucene/newer/NewerLucenePlugin.java deleted file mode 100644 index a948eece809..00000000000 --- a/core/src/test/java/org/elasticsearch/plugins/lucene/newer/NewerLucenePlugin.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.plugins.lucene.newer; - -import org.elasticsearch.plugins.AbstractPlugin; - -public class NewerLucenePlugin extends AbstractPlugin { - /** - * The name of the plugin. - */ - @Override - public String name() { - return "newer-lucene"; - } - - /** - * The description of the plugin. - */ - @Override - public String description() { - return "newer"; - } -} diff --git a/core/src/test/java/org/elasticsearch/plugins/lucene/newer/es-plugin-test.properties b/core/src/test/java/org/elasticsearch/plugins/lucene/newer/es-plugin-test.properties deleted file mode 100644 index 4ddbca8130e..00000000000 --- a/core/src/test/java/org/elasticsearch/plugins/lucene/newer/es-plugin-test.properties +++ /dev/null @@ -1,21 +0,0 @@ -################################################################ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -################################################################ -plugin=org.elasticsearch.plugins.lucene.newer.NewerLucenePlugin -version=3.0.0 -lucene=99.0.0 diff --git a/core/src/test/java/org/elasticsearch/plugins/lucene/old/OldLucenePlugin.java b/core/src/test/java/org/elasticsearch/plugins/lucene/old/OldLucenePlugin.java deleted file mode 100644 index 8f85418c0b1..00000000000 --- a/core/src/test/java/org/elasticsearch/plugins/lucene/old/OldLucenePlugin.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.plugins.lucene.old; - -import org.elasticsearch.plugins.AbstractPlugin; - -public class OldLucenePlugin extends AbstractPlugin { - /** - * The name of the plugin. - */ - @Override - public String name() { - return "old-lucene"; - } - - /** - * The description of the plugin. - */ - @Override - public String description() { - return "old"; - } -} diff --git a/core/src/test/java/org/elasticsearch/plugins/lucene/old/es-plugin-test.properties b/core/src/test/java/org/elasticsearch/plugins/lucene/old/es-plugin-test.properties deleted file mode 100644 index c99f9b1dc6c..00000000000 --- a/core/src/test/java/org/elasticsearch/plugins/lucene/old/es-plugin-test.properties +++ /dev/null @@ -1,21 +0,0 @@ -################################################################ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -################################################################ -plugin=org.elasticsearch.plugins.lucene.old.OldLucenePlugin -version=1.0.0 -lucene=3.0.0 diff --git a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java b/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java index ac785c1601d..05d8fceeee6 100644 --- a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java +++ b/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java @@ -292,7 +292,6 @@ public final class InternalTestCluster extends TestCluster { builder.put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true); builder.put("node.mode", NODE_MODE); builder.put("http.pipelining", enableHttpPipelining); - builder.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false); builder.put(NodeEnvironment.SETTING_CUSTOM_DATA_PATH_ENABLED, true); if (Strings.hasLength(System.getProperty("es.logger.level"))) { builder.put("logger.level", System.getProperty("es.logger.level")); @@ -908,7 +907,6 @@ public final class InternalTestCluster extends TestCluster { .put("client.transport.nodes_sampler_interval", "1s") .put("path.home", baseDir) .put("name", TRANSPORT_CLIENT_PREFIX + node.settings().get("name")) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false) .put(ClusterName.SETTING, clusterName).put("client.transport.sniff", sniff) .put("node.mode", nodeSettings.get("node.mode", NODE_MODE)) .put("node.local", nodeSettings.get("node.local", "")) diff --git a/core/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/core/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index 25cef6db14d..076297c3920 100644 --- a/core/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/core/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -35,7 +35,7 @@ import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.node.info.PluginInfo; +import org.elasticsearch.plugins.PluginInfo; import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo; import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; diff --git a/core/src/test/resources/org/elasticsearch/plugins/loading/jar/in-jar-plugin.jar b/core/src/test/resources/org/elasticsearch/plugins/loading/jar/in-jar-plugin.jar deleted file mode 100644 index 6b854823f143c7bdd70d89922b98b63beb1331da..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2039 zcmWIWW@h1H00FV&4p%S(O0WRw{GxRI0Guj>pej;x5{pYRlZ#Umi;^?&sF%X3z96SG zJu|Nuk3J>5`f~CUQ!?`q?&JXbME~aUr%sFv3>Hib48}z1_ssK3ED8WSMlTuYqvEL( ze7z4l2(;bL{grB;vo^j$VDa*>j)R+(L>xD;2x{;3pS<<%EVb0hzjl97srbPDWAPEs zpe0kP3ivm_KeO-rnGNyx<>xb;+ZgZ2xYlUf=^L7f+gofKA1&nysZmYXV$Iz6r}Twq zW7}e;O7r=AOY4>_JU?}s@UAEP9;;R2MLvIwaAn#i{A!|ZZCBD$w@F*9x26B9S<-ak zoAaj-gH8F(axZ^xsW_P65d5tCmTbC{e#-A7)9TjTJh?v9-)&vk=7`otho$o8wrb~x z7;W5n<++Yyw#Firfbd3B#oW|n*1lb7YG);Po;R5^@8qmAao-Qtcv(mOzZJRQ@Am!H zJxRWKb0Q0SZ2lJ)@pPov_NAW7*~fan^>TTKxtL{6fzGm{%pn%njF?xgIk2nmENhN# z!s4mbv5x%H10IX#u5s)Q{gR@4%O^J49+q?_}u^e>Nb$)%M8CE;-5B8#`}J zQaf4v`TOE0B_FmWml|q(ocR9Z#n)d?pIdEur^xqKF!R9!>)hpcd}~?vF5wEVrNOKN z77{(mD|-3*`jZX(Cj9MT$&tP2U{%2?D%jGuK!gA1RX)Ev!X|IhrX>s7*M)@Eev+DKt4!)kTVVBL?do?}}7|%0ZRXgw^GjR62tWAy_+`A8c z>)EB|@hfP)x2Wok_MKcak{vvctFWKRoXM_nOC{V-yy@?TyMC9K7)%U3w)J*;>Fwy5 z*@18GaXoC=aoM&~|L`x%O_Mq0&o_kqxKbf9&DQVY-ju*i5ga~#%hulRZP;ts%6a#` z+}VBlCp({|HuZ3CT@W70TDi69eB{({=^f|!&7a<7e7E_poi^XkbxlS5;i-jljEX7Ozc&|+n|G)Qxu2SrWml+ssl+}|9&~NVd7K?izTU_n*R7tpYi$9|Ks11Pyd>1u_R+>Z&uxhRpt`m5l??< z?|T38lQ}G9FffRt6+8UEl;P_d;;8HC=cb>VSeBZHR5G9|*2Gi{EVA@a3M4(u0tVd} zBT|fkrddOvvly8~m~j`LKtq8*fZ?wrh=vrUxOIREQwRX*z+U9xvk_FdBJ>9VnIIcc zLP8%A61d#~vKazE?&tzCahroEdST{aFMMGpZfQJ4lzA9MF~Z~6ixPwdvB2^kXTgFn z279`L7$d;&|9?5K_QObpFi&D?2BkxU<|UY#@uf=K#)8r%!r0fO7>krjS=m5la01~n KVBlD@f_MPqy{F~? diff --git a/core/src/test/resources/org/elasticsearch/plugins/plugin_folder_file.zip b/core/src/test/resources/org/elasticsearch/plugins/plugin_folder_file.zip deleted file mode 100644 index 0dbf53d89f37dbaa7bfeffce12be28b052370c8c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 503 zcmWIWW@Zs#U|`^2_%TbydER*z6Imcn1&FyAWEe8@Qc^4QGD>oDLPIzin3av>f>eOG zw1S&~k>x8R0|Qv^=93yH&u)CYN%N%6Nzbj^Cp3M}pV@q|{lktNg-HwnKnu8l#xXGb z23i1SKnZrB1qC^!>6v-@VAC~$Cfj40j@tl1gaIX~#U)^O5-^I9NsbwpcO}5?0(y#J zNh64f?0Hs*=g~YK;0-blm+v6vF)#=)ymd?jng{nQ&@hl+Nia;10okxc2*Z&5jmub8 RHjrFj7)OOxSTHmbOZ

zip false + + + ${elasticsearch.tools.directory}/plugin-metadata/plugin-descriptor.properties + + true + + / diff --git a/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties b/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties new file mode 100644 index 00000000000..b9e7eac1302 --- /dev/null +++ b/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties @@ -0,0 +1,32 @@ +# elasticsearch plugin descriptor file +# +# example: +# jvm=true +# classname=foo.bar.BazPlugin +# isolated=true +# site=false +# description=My cool plugin +# version=2.0 +# elasticsearch.version=2.0 +# +# A plugin can be 'jvm', 'site', or both +# +# 'jvm': true if the 'classname' class should be loaded +# from jar files in the root directory of the plugin +jvm=${elasticsearch.plugin.jvm} +# 'classname': the name of the class to load. +classname=${elasticsearch.plugin.classname} +# 'isolated': true if the plugin should have its own classloader. +# passing false is deprecated, and only intended to support plugins +# that have hard dependencies against each other +isolated=${elasticsearch.plugin.isolated} +# +# 'site': true if the contents of _site should be served +site=${elasticsearch.plugin.site} +# +# 'description': simple summary of the plugin +description=${project.description} +# 'version': plugin's version +version=${project.version} +# 'elasticsearch.version' version of elasticsearch compiled against +elasticsearch.version=${elasticsearch.version} diff --git a/plugins/analysis-icu/pom.xml b/plugins/analysis-icu/pom.xml index f7103c3a979..1214cbc82c0 100644 --- a/plugins/analysis-icu/pom.xml +++ b/plugins/analysis-icu/pom.xml @@ -15,6 +15,7 @@ The ICU Analysis plugin integrates Lucene ICU module into elasticsearch, adding ICU relates analysis components. + org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin analysis_icu false diff --git a/plugins/analysis-icu/src/main/resources/es-plugin.properties b/plugins/analysis-icu/src/main/resources/es-plugin.properties deleted file mode 100644 index 66dd160c014..00000000000 --- a/plugins/analysis-icu/src/main/resources/es-plugin.properties +++ /dev/null @@ -1,3 +0,0 @@ -plugin=org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin -version=${project.version} -lucene=${lucene.version} diff --git a/plugins/analysis-kuromoji/pom.xml b/plugins/analysis-kuromoji/pom.xml index a8014bf6f08..7cb0fa762ea 100644 --- a/plugins/analysis-kuromoji/pom.xml +++ b/plugins/analysis-kuromoji/pom.xml @@ -16,6 +16,7 @@ The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch. + org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin analysis_kuromoji false diff --git a/plugins/analysis-kuromoji/src/main/assemblies/plugin.xml b/plugins/analysis-kuromoji/src/main/assemblies/plugin.xml deleted file mode 100644 index df7f7cb35da..00000000000 --- a/plugins/analysis-kuromoji/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - plugin - - zip - - false - - - / - true - true - true - - org.elasticsearch:elasticsearch - - - - diff --git a/plugins/analysis-kuromoji/src/main/resources/es-plugin.properties b/plugins/analysis-kuromoji/src/main/resources/es-plugin.properties deleted file mode 100644 index ac98c3db5d1..00000000000 --- a/plugins/analysis-kuromoji/src/main/resources/es-plugin.properties +++ /dev/null @@ -1,3 +0,0 @@ -plugin=org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin -version=${project.version} -lucene=${lucene.version} diff --git a/plugins/analysis-phonetic/pom.xml b/plugins/analysis-phonetic/pom.xml index 97db6fa2412..f5bafe2b865 100644 --- a/plugins/analysis-phonetic/pom.xml +++ b/plugins/analysis-phonetic/pom.xml @@ -15,9 +15,10 @@ The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch. + org.elasticsearch.plugin.analysis.AnalysisPhoneticPlugin analysis_phonetic false - + diff --git a/plugins/analysis-phonetic/src/main/assemblies/plugin.xml b/plugins/analysis-phonetic/src/main/assemblies/plugin.xml deleted file mode 100644 index df7f7cb35da..00000000000 --- a/plugins/analysis-phonetic/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - plugin - - zip - - false - - - / - true - true - true - - org.elasticsearch:elasticsearch - - - - diff --git a/plugins/analysis-phonetic/src/main/resources/es-plugin.properties b/plugins/analysis-phonetic/src/main/resources/es-plugin.properties deleted file mode 100644 index cc52b051102..00000000000 --- a/plugins/analysis-phonetic/src/main/resources/es-plugin.properties +++ /dev/null @@ -1,3 +0,0 @@ -plugin=org.elasticsearch.plugin.analysis.AnalysisPhoneticPlugin -version=${project.version} -lucene=${lucene.version} diff --git a/plugins/analysis-smartcn/pom.xml b/plugins/analysis-smartcn/pom.xml index 8145316eb69..178d5f142a6 100644 --- a/plugins/analysis-smartcn/pom.xml +++ b/plugins/analysis-smartcn/pom.xml @@ -15,6 +15,7 @@ Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch. + org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin analysis_smartcn false diff --git a/plugins/analysis-smartcn/src/main/assemblies/plugin.xml b/plugins/analysis-smartcn/src/main/assemblies/plugin.xml deleted file mode 100644 index df7f7cb35da..00000000000 --- a/plugins/analysis-smartcn/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - plugin - - zip - - false - - - / - true - true - true - - org.elasticsearch:elasticsearch - - - - diff --git a/plugins/analysis-smartcn/src/main/resources/es-plugin.properties b/plugins/analysis-smartcn/src/main/resources/es-plugin.properties deleted file mode 100644 index 5da58c07b49..00000000000 --- a/plugins/analysis-smartcn/src/main/resources/es-plugin.properties +++ /dev/null @@ -1,3 +0,0 @@ -plugin=org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin -version=${project.version} -lucene=${lucene.version} diff --git a/plugins/analysis-stempel/pom.xml b/plugins/analysis-stempel/pom.xml index 78f4de95f64..4f8f59c0611 100644 --- a/plugins/analysis-stempel/pom.xml +++ b/plugins/analysis-stempel/pom.xml @@ -15,6 +15,7 @@ The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch. + org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin analysis_stempel false diff --git a/plugins/analysis-stempel/src/main/assemblies/plugin.xml b/plugins/analysis-stempel/src/main/assemblies/plugin.xml deleted file mode 100644 index df7f7cb35da..00000000000 --- a/plugins/analysis-stempel/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - plugin - - zip - - false - - - / - true - true - true - - org.elasticsearch:elasticsearch - - - - diff --git a/plugins/analysis-stempel/src/main/resources/es-plugin.properties b/plugins/analysis-stempel/src/main/resources/es-plugin.properties deleted file mode 100644 index 00dd859377a..00000000000 --- a/plugins/analysis-stempel/src/main/resources/es-plugin.properties +++ /dev/null @@ -1,3 +0,0 @@ -plugin=org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin -version=${project.version} -lucene=${lucene.version} diff --git a/plugins/cloud-aws/pom.xml b/plugins/cloud-aws/pom.xml index f990a565580..35edcf5d238 100644 --- a/plugins/cloud-aws/pom.xml +++ b/plugins/cloud-aws/pom.xml @@ -15,6 +15,7 @@ The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories. + org.elasticsearch.plugin.cloud.aws.CloudAwsPlugin 1.10.0 1 cloud_aws diff --git a/plugins/cloud-aws/src/main/assemblies/plugin.xml b/plugins/cloud-aws/src/main/assemblies/plugin.xml deleted file mode 100644 index df7f7cb35da..00000000000 --- a/plugins/cloud-aws/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - plugin - - zip - - false - - - / - true - true - true - - org.elasticsearch:elasticsearch - - - - diff --git a/plugins/cloud-aws/src/main/resources/es-plugin.properties b/plugins/cloud-aws/src/main/resources/es-plugin.properties deleted file mode 100644 index 234aa512e2e..00000000000 --- a/plugins/cloud-aws/src/main/resources/es-plugin.properties +++ /dev/null @@ -1,3 +0,0 @@ -plugin=org.elasticsearch.plugin.cloud.aws.CloudAwsPlugin -version=${project.version} - diff --git a/plugins/cloud-aws/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java b/plugins/cloud-aws/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java index 974bc96a900..a31a6525a7c 100644 --- a/plugins/cloud-aws/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java +++ b/plugins/cloud-aws/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.FailedToResolveConfigException; +import org.elasticsearch.plugin.cloud.aws.CloudAwsPlugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ThirdParty; @@ -81,7 +82,7 @@ public abstract class AbstractAwsTest extends ElasticsearchIntegrationTest { Settings.Builder settings = Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put("path.home", createTempDir()) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("plugin.types", CloudAwsPlugin.class.getName()) .put(AwsModule.S3_SERVICE_TYPE_KEY, TestAwsS3Service.class) .put("cloud.aws.test.random", randomInt()) .put("cloud.aws.test.write_failures", 0.1) diff --git a/plugins/cloud-aws/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java b/plugins/cloud-aws/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java index f1c44063500..c450a5920b2 100644 --- a/plugins/cloud-aws/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java +++ b/plugins/cloud-aws/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java @@ -22,6 +22,7 @@ package org.elasticsearch.discovery.ec2; import org.elasticsearch.cloud.aws.AbstractAwsTest; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugin.cloud.aws.CloudAwsPlugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; @@ -40,7 +41,7 @@ public class Ec2DiscoveryITest extends AbstractAwsTest { @Test public void testStart() { Settings nodeSettings = settingsBuilder() - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("plugin.types", CloudAwsPlugin.class.getName()) .put("cloud.enabled", true) .put("discovery.type", "ec2") .build(); diff --git a/plugins/cloud-aws/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java b/plugins/cloud-aws/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java index 6d74bcbaf92..2af8b895314 100644 --- a/plugins/cloud-aws/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java +++ b/plugins/cloud-aws/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java @@ -23,6 +23,7 @@ package org.elasticsearch.discovery.ec2; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.elasticsearch.cloud.aws.AbstractAwsTest; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugin.cloud.aws.CloudAwsPlugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; @@ -42,7 +43,7 @@ public class Ec2DiscoveryUpdateSettingsITest extends AbstractAwsTest { @Test public void testMinimumMasterNodesStart() { Settings nodeSettings = settingsBuilder() - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("plugin.types", CloudAwsPlugin.class.getName()) .put("cloud.enabled", true) .put("discovery.type", "ec2") .build(); diff --git a/plugins/cloud-aws/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/cloud-aws/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index af8cfb11780..df619b8af37 100644 --- a/plugins/cloud-aws/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/plugins/cloud-aws/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -33,6 +33,7 @@ import org.elasticsearch.cloud.aws.AbstractAwsTest; import org.elasticsearch.cloud.aws.AwsS3Service; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugin.cloud.aws.CloudAwsPlugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.repositories.RepositoryVerificationException; @@ -63,7 +64,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest { .put(MockFSDirectoryService.RANDOM_PREVENT_DOUBLE_WRITE, false) .put(MockFSDirectoryService.RANDOM_NO_DELETE_OPEN_FILE, false) .put("cloud.enabled", true) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("plugin.types", CloudAwsPlugin.class.getName()) .build(); } diff --git a/plugins/cloud-azure/pom.xml b/plugins/cloud-azure/pom.xml index 22d1c860a32..c339c114cb1 100644 --- a/plugins/cloud-azure/pom.xml +++ b/plugins/cloud-azure/pom.xml @@ -26,7 +26,7 @@ governing permissions and limitations under the License. --> The Azure Cloud plugin allows to use Azure API for the unicast discovery mechanism and add Azure storage repositories. - + org.elasticsearch.plugin.cloud.azure.CloudAzurePlugin 1 cloud_azure false diff --git a/plugins/cloud-azure/src/main/assemblies/plugin.xml b/plugins/cloud-azure/src/main/assemblies/plugin.xml deleted file mode 100644 index df7f7cb35da..00000000000 --- a/plugins/cloud-azure/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - plugin - - zip - - false - - - / - true - true - true - - org.elasticsearch:elasticsearch - - - - diff --git a/plugins/cloud-azure/src/main/resources/es-plugin.properties b/plugins/cloud-azure/src/main/resources/es-plugin.properties deleted file mode 100644 index 3bcf1e294a7..00000000000 --- a/plugins/cloud-azure/src/main/resources/es-plugin.properties +++ /dev/null @@ -1,12 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with this work for additional -# information regarding copyright ownership. ElasticSearch licenses this file to you -# under the Apache License, Version 2.0 (the "License"); you may not use this -# file except in compliance with the License. You may obtain a copy of the -# License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by -# applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the specific language -# governing permissions and limitations under the License. -plugin=org.elasticsearch.plugin.cloud.azure.CloudAzurePlugin -version=${project.version} diff --git a/plugins/cloud-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTest.java b/plugins/cloud-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTest.java index 7a2cc37d136..c366db9ab19 100644 --- a/plugins/cloud-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTest.java +++ b/plugins/cloud-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTest.java @@ -23,7 +23,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.FailedToResolveConfigException; -import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.plugin.cloud.azure.CloudAzurePlugin; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ThirdParty; @@ -40,7 +40,7 @@ public abstract class AbstractAzureTest extends ElasticsearchIntegrationTest { protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("plugin.types", CloudAzurePlugin.class.getName()) .put(readSettingsFromFile()) .build(); } diff --git a/plugins/cloud-azure/src/test/java/org/elasticsearch/discovery/azure/AbstractAzureComputeServiceTest.java b/plugins/cloud-azure/src/test/java/org/elasticsearch/discovery/azure/AbstractAzureComputeServiceTest.java index cf3b7848d0e..106f51383d1 100644 --- a/plugins/cloud-azure/src/test/java/org/elasticsearch/discovery/azure/AbstractAzureComputeServiceTest.java +++ b/plugins/cloud-azure/src/test/java/org/elasticsearch/discovery/azure/AbstractAzureComputeServiceTest.java @@ -24,7 +24,7 @@ import org.elasticsearch.cloud.azure.management.AzureComputeService; import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery; import org.elasticsearch.cloud.azure.management.AzureComputeService.Management; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.plugin.cloud.azure.CloudAzurePlugin; import org.elasticsearch.test.ElasticsearchIntegrationTest; public abstract class AbstractAzureComputeServiceTest extends ElasticsearchIntegrationTest { @@ -40,7 +40,7 @@ public abstract class AbstractAzureComputeServiceTest extends ElasticsearchInteg protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder settings = Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true); + .put("plugin.types", CloudAzurePlugin.class.getName()); return settings.build(); } diff --git a/plugins/cloud-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java b/plugins/cloud-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java index 56bad4cd0e3..876430213e4 100644 --- a/plugins/cloud-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java +++ b/plugins/cloud-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java @@ -20,11 +20,13 @@ package org.elasticsearch.repositories.azure; import com.microsoft.azure.storage.StorageException; + import org.elasticsearch.cloud.azure.AbstractAzureTest; import org.elasticsearch.cloud.azure.storage.AzureStorageService; import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugin.cloud.azure.CloudAzurePlugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.test.store.MockFSDirectoryService; @@ -65,7 +67,7 @@ public abstract class AbstractAzureRepositoryServiceTest extends AbstractAzureTe @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.settingsBuilder() - .put(PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("plugin.types", CloudAzurePlugin.class.getName()) .put(Storage.API_IMPLEMENTATION, mock) .put(Storage.CONTAINER, "snapshots"); diff --git a/plugins/cloud-gce/pom.xml b/plugins/cloud-gce/pom.xml index 65ca80cc506..48a7b26542e 100644 --- a/plugins/cloud-gce/pom.xml +++ b/plugins/cloud-gce/pom.xml @@ -26,6 +26,7 @@ governing permissions and limitations under the License. --> The Google Compute Engine (GCE) Cloud plugin allows to use GCE API for the unicast discovery mechanism. + org.elasticsearch.plugin.cloud.gce.CloudGcePlugin v1-rev59-1.20.0 9300 diff --git a/plugins/cloud-gce/src/main/assemblies/plugin.xml b/plugins/cloud-gce/src/main/assemblies/plugin.xml deleted file mode 100644 index df7f7cb35da..00000000000 --- a/plugins/cloud-gce/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - plugin - - zip - - false - - - / - true - true - true - - org.elasticsearch:elasticsearch - - - - diff --git a/plugins/cloud-gce/src/main/resources/es-plugin.properties b/plugins/cloud-gce/src/main/resources/es-plugin.properties deleted file mode 100644 index e6dfb1b8d78..00000000000 --- a/plugins/cloud-gce/src/main/resources/es-plugin.properties +++ /dev/null @@ -1,12 +0,0 @@ -# Licensed to ElasticSearch under one or more contributor -# license agreements. See the NOTICE file distributed with this work for additional -# information regarding copyright ownership. ElasticSearch licenses this file to you -# under the Apache License, Version 2.0 (the "License"); you may not use this -# file except in compliance with the License. You may obtain a copy of the -# License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by -# applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the specific language -# governing permissions and limitations under the License. -plugin=org.elasticsearch.plugin.cloud.gce.CloudGcePlugin -version=${project.version} diff --git a/plugins/cloud-gce/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java index cc34726264a..fe648be5391 100644 --- a/plugins/cloud-gce/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java +++ b/plugins/cloud-gce/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.FailedToResolveConfigException; +import org.elasticsearch.plugin.cloud.gce.CloudGcePlugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ThirdParty; @@ -38,7 +39,7 @@ public abstract class AbstractGceTest extends ElasticsearchIntegrationTest { Settings.Builder settings = Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put("path.home", createTempDir()) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true); + .put("plugin.types", CloudGcePlugin.class.getName()); Environment environment = new Environment(settings.build()); diff --git a/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java index 2bbfa60cfe1..fe231cabd1a 100644 --- a/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java +++ b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java @@ -20,11 +20,13 @@ package org.elasticsearch.discovery.gce; import com.google.common.collect.Lists; + import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.cloud.gce.GceComputeService.Fields; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.gce.mock.*; +import org.elasticsearch.plugin.cloud.gce.CloudGcePlugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.junit.Ignore; @@ -73,7 +75,7 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { // We disable http .put("http.enabled", false) // We force plugin loading - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("plugin.types", CloudGcePlugin.class.getName()) .put(settings) .put(super.nodeSettings(nodeOrdinal)); diff --git a/plugins/cloud-gce/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java index 67ecbee8592..a6b77538f64 100644 --- a/plugins/cloud-gce/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java +++ b/plugins/cloud-gce/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java @@ -22,6 +22,7 @@ package org.elasticsearch.gce.itest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.cloud.gce.AbstractGceTest; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugin.cloud.gce.CloudGcePlugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.hamcrest.Matchers; @@ -44,7 +45,7 @@ public class GceSimpleITest extends AbstractGceTest { protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put("plugin.types", CloudGcePlugin.class.getName()) .build(); } diff --git a/plugins/delete-by-query/.gitignore b/plugins/delete-by-query/.gitignore new file mode 100644 index 00000000000..ae3c1726048 --- /dev/null +++ b/plugins/delete-by-query/.gitignore @@ -0,0 +1 @@ +/bin/ diff --git a/plugins/delete-by-query/pom.xml b/plugins/delete-by-query/pom.xml index 02ef5206940..62ee8fdb9b1 100644 --- a/plugins/delete-by-query/pom.xml +++ b/plugins/delete-by-query/pom.xml @@ -26,6 +26,7 @@ governing permissions and limitations under the License. --> The Delete By Query plugin allows to delete documents in Elasticsearch with a single query. + org.elasticsearch.plugin.deletebyquery.DeleteByQueryPlugin warn delete_by_query false diff --git a/plugins/delete-by-query/src/main/assemblies/plugin.xml b/plugins/delete-by-query/src/main/assemblies/plugin.xml deleted file mode 100644 index d5a4e719ce8..00000000000 --- a/plugins/delete-by-query/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - plugin - - zip - - false - - - / - true - true - - org.elasticsearch:elasticsearch - - - - / - true - true - - - \ No newline at end of file diff --git a/plugins/delete-by-query/src/main/resources/es-plugin.properties b/plugins/delete-by-query/src/main/resources/es-plugin.properties deleted file mode 100644 index 400411ffac3..00000000000 --- a/plugins/delete-by-query/src/main/resources/es-plugin.properties +++ /dev/null @@ -1,3 +0,0 @@ -plugin=org.elasticsearch.plugin.deletebyquery.DeleteByQueryPlugin -version=${project.version} -lucene=${lucene.version} diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java index 16b96be1793..db3f1f8d104 100644 --- a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java +++ b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java @@ -54,6 +54,13 @@ import static org.hamcrest.Matchers.nullValue; @Slow @ClusterScope(scope = SUITE, transportClientRatio = 0) public class DeleteByQueryTests extends ElasticsearchIntegrationTest { + + protected Settings nodeSettings(int nodeOrdinal) { + Settings.Builder settings = Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugin.types", DeleteByQueryPlugin.class.getName()); + return settings.build(); + } @Test(expected = ActionRequestValidationException.class) public void testDeleteByQueryWithNoSource() { diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestTests.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestTests.java index 31bd3b6db31..d08e5abaae1 100644 --- a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestTests.java +++ b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestTests.java @@ -21,7 +21,10 @@ package org.elasticsearch.plugin.deletebyquery.test.rest; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugin.deletebyquery.DeleteByQueryPlugin; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.rest.ElasticsearchRestTestCase; import org.elasticsearch.test.rest.ElasticsearchRestTestCase.Rest; @@ -46,5 +49,12 @@ public class DeleteByQueryRestTests extends ElasticsearchRestTestCase { public static Iterable parameters() throws IOException, RestTestParseException { return ElasticsearchRestTestCase.createParameters(0, 1); } + + protected Settings nodeSettings(int nodeOrdinal) { + Settings.Builder settings = Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugin.types", DeleteByQueryPlugin.class.getName()); + return settings.build(); + } } diff --git a/plugins/lang-javascript/pom.xml b/plugins/lang-javascript/pom.xml index 74cc225ffe7..eb6e21cd704 100644 --- a/plugins/lang-javascript/pom.xml +++ b/plugins/lang-javascript/pom.xml @@ -15,7 +15,7 @@ The JavaScript language plugin allows to have javascript as the language of scripts to execute. - + org.elasticsearch.plugin.javascript.JavaScriptPlugin lang_javascript false diff --git a/plugins/lang-javascript/src/main/assemblies/plugin.xml b/plugins/lang-javascript/src/main/assemblies/plugin.xml deleted file mode 100644 index df7f7cb35da..00000000000 --- a/plugins/lang-javascript/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - plugin - - zip - - false - - - / - true - true - true - - org.elasticsearch:elasticsearch - - - - diff --git a/plugins/lang-javascript/src/main/resources/es-plugin.properties b/plugins/lang-javascript/src/main/resources/es-plugin.properties deleted file mode 100644 index 92f26b778d7..00000000000 --- a/plugins/lang-javascript/src/main/resources/es-plugin.properties +++ /dev/null @@ -1,2 +0,0 @@ -plugin=org.elasticsearch.plugin.javascript.JavaScriptPlugin -version=${project.version} diff --git a/plugins/lang-python/pom.xml b/plugins/lang-python/pom.xml index fb4529571b3..b69a07b0ed7 100644 --- a/plugins/lang-python/pom.xml +++ b/plugins/lang-python/pom.xml @@ -15,7 +15,7 @@ The Python language plugin allows to have python as the language of scripts to execute. - + org.elasticsearch.plugin.python.PythonPlugin lang_python false diff --git a/plugins/lang-python/src/main/assemblies/plugin.xml b/plugins/lang-python/src/main/assemblies/plugin.xml deleted file mode 100644 index df7f7cb35da..00000000000 --- a/plugins/lang-python/src/main/assemblies/plugin.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - plugin - - zip - - false - - - / - true - true - true - - org.elasticsearch:elasticsearch - - - - diff --git a/plugins/lang-python/src/main/resources/es-plugin.properties b/plugins/lang-python/src/main/resources/es-plugin.properties deleted file mode 100644 index b0ed0d5e3e0..00000000000 --- a/plugins/lang-python/src/main/resources/es-plugin.properties +++ /dev/null @@ -1,2 +0,0 @@ -plugin=org.elasticsearch.plugin.python.PythonPlugin -version=${project.version} diff --git a/plugins/pom.xml b/plugins/pom.xml index 7f72dbeade3..ad25cb91d5d 100644 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -19,8 +19,11 @@ - ${basedir}/src/main/assemblies/plugin.xml + ${elasticsearch.tools.directory}/plugin-metadata/plugin-assembly.xml false + true + true + false @@ -349,6 +352,25 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-plugin-classname + + enforce + + + + + elasticsearch.plugin.classname + + + + + + diff --git a/pom.xml b/pom.xml index 090d5708a0e..e74b95d5bc1 100644 --- a/pom.xml +++ b/pom.xml @@ -1349,6 +1349,7 @@ org.eclipse.jdt.ui.text.custom_code_templates= true true + true From aba1ec4580d1df6e586998e0464e5cb1c4df8cea Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 22 Jul 2015 12:42:15 -0400 Subject: [PATCH 5/9] skip this download/copy unless it will be used. this makes less noise and disk space for the POM builds. --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index e74b95d5bc1..5df506568e6 100644 --- a/pom.xml +++ b/pom.xml @@ -909,6 +909,7 @@ copy + ${skip.integ.tests} org.elasticsearch From a33cfe4b11c55652eb04980664e0d2d973903f93 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 22 Jul 2015 11:15:57 -0600 Subject: [PATCH 6/9] Revert "Consistently name Groovy scripts with the same content" This reverts commit d902012835916179e0bb86890e7898cc30aef70a. Reverting this temporarily to see if this fixes hitting an assert in the JDK during Groovy's classloading --- .../script/groovy/GroovyScriptEngineService.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java index 17c4284f714..683ffc58f0f 100644 --- a/core/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java +++ b/core/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java @@ -19,8 +19,6 @@ package org.elasticsearch.script.groovy; -import com.google.common.base.Charsets; -import com.google.common.hash.Hashing; import groovy.lang.Binding; import groovy.lang.GroovyClassLoader; import groovy.lang.Script; @@ -51,6 +49,7 @@ import java.io.IOException; import java.math.BigDecimal; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; /** * Provides the infrastructure for Groovy as a scripting language for Elasticsearch @@ -58,6 +57,7 @@ import java.util.Map; public class GroovyScriptEngineService extends AbstractComponent implements ScriptEngineService { public static final String NAME = "groovy"; + private final AtomicLong counter = new AtomicLong(); private final GroovyClassLoader loader; @Inject @@ -111,7 +111,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri @Override public Object compile(String script) { try { - return loader.parseClass(script, Hashing.sha1().hashString(script, Charsets.UTF_8).toString()); + return loader.parseClass(script, generateScriptName()); } catch (Throwable e) { if (logger.isTraceEnabled()) { logger.trace("exception compiling Groovy script:", e); @@ -190,6 +190,10 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri return value; } + private String generateScriptName() { + return "Script" + counter.incrementAndGet() + ".groovy"; + } + public static final class GroovyScript implements ExecutableScript, LeafSearchScript { private final CompiledScript compiledScript; From 33d2ca13a9288dae7e8da2248f112b424a7fbebb Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 22 Jul 2015 18:42:31 +0200 Subject: [PATCH 7/9] Simplify Replica Allocator Simplify the codebase of replica allocator and add more unit tests for it --- .../gateway/ReplicaShardAllocator.java | 257 +++++++++++------- .../gateway/ReplicaShardAllocatorTests.java | 37 ++- 2 files changed, 187 insertions(+), 107 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java index 6879bda4fa2..2d0d38d0bd2 100644 --- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java +++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java @@ -19,7 +19,10 @@ package org.elasticsearch.gateway; +import com.carrotsearch.hppc.ObjectLongHashMap; +import com.carrotsearch.hppc.ObjectLongMap; import com.carrotsearch.hppc.cursors.ObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectLongCursor; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -29,6 +32,7 @@ import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; @@ -50,8 +54,6 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { public boolean allocateUnassigned(RoutingAllocation allocation) { boolean changed = false; final RoutingNodes routingNodes = allocation.routingNodes(); - final MetaData metaData = routingNodes.metaData(); - final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = routingNodes.unassigned().iterator(); while (unassignedIterator.hasNext()) { ShardRouting shard = unassignedIterator.next(); @@ -60,22 +62,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { } // pre-check if it can be allocated to any node that currently exists, so we won't list the store for it for nothing - boolean canBeAllocatedToAtLeastOneNode = false; - for (ObjectCursor cursor : allocation.nodes().dataNodes().values()) { - RoutingNode node = routingNodes.node(cursor.value.id()); - if (node == null) { - continue; - } - // if we can't allocate it on a node, ignore it, for example, this handles - // cases for only allocating a replica after a primary - Decision decision = allocation.deciders().canAllocate(shard, node, allocation); - if (decision.type() == Decision.Type.YES) { - canBeAllocatedToAtLeastOneNode = true; - break; - } - } - - if (!canBeAllocatedToAtLeastOneNode) { + if (canBeAllocatedToAtLeastOneNode(shard, allocation) == false) { logger.trace("{}: ignoring allocation, can't be allocated on any node", shard); unassignedIterator.removeAndIgnore(); continue; @@ -88,106 +75,41 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { continue; // still fetching } - long lastSizeMatched = 0; - DiscoveryNode lastDiscoNodeMatched = null; - RoutingNode lastNodeMatched = null; - boolean hasReplicaData = false; - IndexMetaData indexMetaData = metaData.index(shard.getIndex()); - - for (Map.Entry nodeStoreEntry : shardStores.getData().entrySet()) { - DiscoveryNode discoNode = nodeStoreEntry.getKey(); - TransportNodesListShardStoreMetaData.StoreFilesMetaData storeFilesMetaData = nodeStoreEntry.getValue().storeFilesMetaData(); - logger.trace("{}: checking node [{}]", shard, discoNode); - - if (storeFilesMetaData == null) { - // already allocated on that node... - continue; - } - - RoutingNode node = routingNodes.node(discoNode.id()); - if (node == null) { - continue; - } - - // check if we can allocate on that node... - // we only check for NO, since if this node is THROTTLING and it has enough "same data" - // then we will try and assign it next time - Decision decision = allocation.deciders().canAllocate(shard, node, allocation); - if (decision.type() == Decision.Type.NO) { - continue; - } - - // if it is already allocated, we can't assign to it... - if (storeFilesMetaData.allocated()) { - continue; - } - - if (!shard.primary()) { - hasReplicaData |= storeFilesMetaData.iterator().hasNext(); - ShardRouting primaryShard = routingNodes.activePrimary(shard); - if (primaryShard != null) { - assert primaryShard.active(); - DiscoveryNode primaryNode = allocation.nodes().get(primaryShard.currentNodeId()); - if (primaryNode != null) { - TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData primaryNodeFilesStore = shardStores.getData().get(primaryNode); - if (primaryNodeFilesStore != null) { - TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryNodeStore = primaryNodeFilesStore.storeFilesMetaData(); - if (primaryNodeStore != null && primaryNodeStore.allocated()) { - long sizeMatched = 0; - - String primarySyncId = primaryNodeStore.syncId(); - String replicaSyncId = storeFilesMetaData.syncId(); - // see if we have a sync id we can make use of - if (replicaSyncId != null && replicaSyncId.equals(primarySyncId)) { - logger.trace("{}: node [{}] has same sync id {} as primary", shard, discoNode.name(), replicaSyncId); - lastNodeMatched = node; - lastSizeMatched = Long.MAX_VALUE; - lastDiscoNodeMatched = discoNode; - } else { - for (StoreFileMetaData storeFileMetaData : storeFilesMetaData) { - String metaDataFileName = storeFileMetaData.name(); - if (primaryNodeStore.fileExists(metaDataFileName) && primaryNodeStore.file(metaDataFileName).isSame(storeFileMetaData)) { - sizeMatched += storeFileMetaData.length(); - } - } - logger.trace("{}: node [{}] has [{}/{}] bytes of re-usable data", - shard, discoNode.name(), new ByteSizeValue(sizeMatched), sizeMatched); - if (sizeMatched > lastSizeMatched) { - lastSizeMatched = sizeMatched; - lastDiscoNodeMatched = discoNode; - lastNodeMatched = node; - } - } - } - } - } - } - } + ShardRouting primaryShard = routingNodes.activePrimary(shard); + assert primaryShard != null : "the replica shard can be allocated on at least one node, so there must be an active primary"; + TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore = findStore(primaryShard, allocation, shardStores); + if (primaryStore == null || primaryStore.allocated() == false) { + // if we can't find the primary data, it is probably because the primary shard is corrupted (and listing failed) + // we want to let the replica be allocated in order to expose the actual problem with the primary that the replica + // will try and recover from + // Note, this is the existing behavior, as exposed in running CorruptFileTest#testNoPrimaryData + logger.trace("{}: no primary shard store found or allocated, letting actual allocation figure it out", shard); + continue; } - if (lastNodeMatched != null) { + MatchingNodes matchingNodes = findMatchingNodes(shard, allocation, primaryStore, shardStores); + + if (matchingNodes.getNodeWithHighestMatch() != null) { + RoutingNode nodeWithHighestMatch = allocation.routingNodes().node(matchingNodes.getNodeWithHighestMatch().id()); // we only check on THROTTLE since we checked before before on NO - Decision decision = allocation.deciders().canAllocate(shard, lastNodeMatched, allocation); + Decision decision = allocation.deciders().canAllocate(shard, nodeWithHighestMatch, allocation); if (decision.type() == Decision.Type.THROTTLE) { - if (logger.isDebugEnabled()) { - logger.debug("[{}][{}]: throttling allocation [{}] to [{}] in order to reuse its unallocated persistent store with total_size [{}]", shard.index(), shard.id(), shard, lastDiscoNodeMatched, new ByteSizeValue(lastSizeMatched)); - } + logger.debug("[{}][{}]: throttling allocation [{}] to [{}] in order to reuse its unallocated persistent store", shard.index(), shard.id(), shard, nodeWithHighestMatch.node()); // we are throttling this, but we have enough to allocate to this node, ignore it for now unassignedIterator.removeAndIgnore(); } else { - if (logger.isDebugEnabled()) { - logger.debug("[{}][{}]: allocating [{}] to [{}] in order to reuse its unallocated persistent store with total_size [{}]", shard.index(), shard.id(), shard, lastDiscoNodeMatched, new ByteSizeValue(lastSizeMatched)); - } + logger.debug("[{}][{}]: allocating [{}] to [{}] in order to reuse its unallocated persistent store", shard.index(), shard.id(), shard, nodeWithHighestMatch.node()); // we found a match changed = true; - unassignedIterator.initialize(lastNodeMatched.nodeId()); + unassignedIterator.initialize(nodeWithHighestMatch.nodeId()); } - } else if (hasReplicaData == false) { + } else if (matchingNodes.hasAnyData() == false) { // if we didn't manage to find *any* data (regardless of matching sizes), check if the allocation // of the replica shard needs to be delayed, and if so, add it to the ignore unassigned list // note: we only care about replica in delayed allocation, since if we have an unassigned primary it // will anyhow wait to find an existing copy of the shard to be allocated // note: the other side of the equation is scheduling a reroute in a timely manner, which happens in the RoutingService + IndexMetaData indexMetaData = allocation.metaData().index(shard.getIndex()); long delay = shard.unassignedInfo().getDelayAllocationExpirationIn(settings, indexMetaData.getSettings()); if (delay > 0) { logger.debug("[{}][{}]: delaying allocation of [{}] for [{}]", shard.index(), shard.id(), shard, TimeValue.timeValueMillis(delay)); @@ -203,5 +125,134 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { return changed; } + /** + * Can the shard be allocated on at least one node based on the allocation deciders. + */ + private boolean canBeAllocatedToAtLeastOneNode(ShardRouting shard, RoutingAllocation allocation) { + for (ObjectCursor cursor : allocation.nodes().dataNodes().values()) { + RoutingNode node = allocation.routingNodes().node(cursor.value.id()); + if (node == null) { + continue; + } + // if we can't allocate it on a node, ignore it, for example, this handles + // cases for only allocating a replica after a primary + Decision decision = allocation.deciders().canAllocate(shard, node, allocation); + if (decision.type() == Decision.Type.YES) { + return true; + } + } + return false; + } + + /** + * Finds the store for the assigned shard in the fetched data, returns null if none is found. + */ + private TransportNodesListShardStoreMetaData.StoreFilesMetaData findStore(ShardRouting shard, RoutingAllocation allocation, AsyncShardFetch.FetchResult data) { + assert shard.currentNodeId() != null; + DiscoveryNode primaryNode = allocation.nodes().get(shard.currentNodeId()); + if (primaryNode == null) { + return null; + } + TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData primaryNodeFilesStore = data.getData().get(primaryNode); + if (primaryNodeFilesStore == null) { + return null; + } + return primaryNodeFilesStore.storeFilesMetaData(); + } + + private MatchingNodes findMatchingNodes(ShardRouting shard, RoutingAllocation allocation, + TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore, + AsyncShardFetch.FetchResult data) { + ObjectLongMap nodesToSize = new ObjectLongHashMap<>(); + for (Map.Entry nodeStoreEntry : data.getData().entrySet()) { + DiscoveryNode discoNode = nodeStoreEntry.getKey(); + TransportNodesListShardStoreMetaData.StoreFilesMetaData storeFilesMetaData = nodeStoreEntry.getValue().storeFilesMetaData(); + if (storeFilesMetaData == null) { + // already allocated on that node... + continue; + } + + RoutingNode node = allocation.routingNodes().node(discoNode.id()); + if (node == null) { + continue; + } + + // check if we can allocate on that node... + // we only check for NO, since if this node is THROTTLING and it has enough "same data" + // then we will try and assign it next time + Decision decision = allocation.deciders().canAllocate(shard, node, allocation); + if (decision.type() == Decision.Type.NO) { + continue; + } + + // if it is already allocated, we can't assign to it... (and it might be primary as well) + if (storeFilesMetaData.allocated()) { + continue; + } + + // we don't have any files at all, it is an empty index + if (storeFilesMetaData.iterator().hasNext() == false) { + continue; + } + + String primarySyncId = primaryStore.syncId(); + String replicaSyncId = storeFilesMetaData.syncId(); + // see if we have a sync id we can make use of + if (replicaSyncId != null && replicaSyncId.equals(primarySyncId)) { + logger.trace("{}: node [{}] has same sync id {} as primary", shard, discoNode.name(), replicaSyncId); + nodesToSize.put(discoNode, Long.MAX_VALUE); + } else { + long sizeMatched = 0; + for (StoreFileMetaData storeFileMetaData : storeFilesMetaData) { + String metaDataFileName = storeFileMetaData.name(); + if (primaryStore.fileExists(metaDataFileName) && primaryStore.file(metaDataFileName).isSame(storeFileMetaData)) { + sizeMatched += storeFileMetaData.length(); + } + } + logger.trace("{}: node [{}] has [{}/{}] bytes of re-usable data", + shard, discoNode.name(), new ByteSizeValue(sizeMatched), sizeMatched); + nodesToSize.put(discoNode, sizeMatched); + } + } + + return new MatchingNodes(nodesToSize); + } + protected abstract AsyncShardFetch.FetchResult fetchData(ShardRouting shard, RoutingAllocation allocation); + + static class MatchingNodes { + private final ObjectLongMap nodesToSize; + private final DiscoveryNode nodeWithHighestMatch; + + public MatchingNodes(ObjectLongMap nodesToSize) { + this.nodesToSize = nodesToSize; + + long highestMatchSize = 0; + DiscoveryNode highestMatchNode = null; + + for (ObjectLongCursor cursor : nodesToSize) { + if (cursor.value > highestMatchSize) { + highestMatchSize = cursor.value; + highestMatchNode = cursor.key; + } + } + nodeWithHighestMatch = highestMatchNode; + } + + /** + * Returns the node with the highest "non zero byte" match compared to + * the primary. + */ + @Nullable + public DiscoveryNode getNodeWithHighestMatch() { + return this.nodeWithHighestMatch; + } + + /** + * Did we manage to find any data, regardless how well they matched or not. + */ + public boolean hasAnyData() { + return nodesToSize.isEmpty() == false; + } + } } diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java index 984adacff75..df78028750b 100644 --- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.Decision; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.Store; @@ -118,8 +119,9 @@ public class ReplicaShardAllocatorTests extends ElasticsearchAllocationTestCase /** * When we can't find primary data, but still find replica data, we go ahead and keep it unassigned - * to be allocated. - * TODO: this might be the wrong decision here, and we should restart the fetching process maybe to really find a primary copy? + * to be allocated. This is today behavior, which relies on a primary corruption identified with + * adding a replica and having that replica actually recover and cause the corruption to be identified + * See CorruptFileTest# */ @Test public void testNoPrimaryData() { @@ -194,15 +196,42 @@ public class ReplicaShardAllocatorTests extends ElasticsearchAllocationTestCase assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId)); } + @Test + public void testDelayedAllocation() { + RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(), + Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, TimeValue.timeValueHours(1)).build(), UnassignedInfo.Reason.NODE_LEFT); + testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM")); + if (randomBoolean()) { + // we sometime return empty list of files, make sure we test this as well + testAllocator.addData(node2, false, null); + } + boolean changed = testAllocator.allocateUnassigned(allocation); + assertThat(changed, equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); + assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId)); + + allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(), + Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, TimeValue.timeValueHours(1)).build(), UnassignedInfo.Reason.NODE_LEFT); + testAllocator.addData(node2, false, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM")); + changed = testAllocator.allocateUnassigned(allocation); + assertThat(changed, equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(node2.id())); + } + private RoutingAllocation onePrimaryOnNode1And1Replica(AllocationDeciders deciders) { + return onePrimaryOnNode1And1Replica(deciders, Settings.EMPTY, UnassignedInfo.Reason.INDEX_CREATED); + } + + private RoutingAllocation onePrimaryOnNode1And1Replica(AllocationDeciders deciders, Settings settings, UnassignedInfo.Reason reason) { MetaData metaData = MetaData.builder() - .put(IndexMetaData.builder(shardId.getIndex()).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)) + .put(IndexMetaData.builder(shardId.getIndex()).settings(settings(Version.CURRENT).put(settings)).numberOfShards(1).numberOfReplicas(0)) .build(); RoutingTable routingTable = RoutingTable.builder() .add(IndexRoutingTable.builder(shardId.getIndex()) .addIndexShard(new IndexShardRoutingTable.Builder(shardId) .addShard(TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node1.id(), true, ShardRoutingState.STARTED, 10)) - .addShard(ShardRouting.newUnassigned(shardId.getIndex(), shardId.getId(), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null))) + .addShard(ShardRouting.newUnassigned(shardId.getIndex(), shardId.getId(), null, false, new UnassignedInfo(reason, null))) .build()) ) .build(); From d78cd66b519cd130402b77d37f6ddcf57850aae8 Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Tue, 21 Jul 2015 22:06:55 +0200 Subject: [PATCH 8/9] [Test] make cluster state blocking more reliable in IndicesStoreIntegrationTests.indexCleanup() IndicesStoreIntegrationTests.indexCleanup() tests if the shard files on disk are actually removed after relocation. In particular it tests the following: Whenever a node deletes a shard because it was relocated somewhere else, it first checks if enough other copies are started somewhere else. The node sends a ShardActiveRequest to the nodes that should have a copy. The nodes that receive this request check if the shard is in state STARTED in which case they respond with true. If they have the shard in POST_RECOVERY they register a cluster state observer that checks at each update if the shard has moved to STARTED and respond with true when this happens. To test that the cluster state observer mechanism actually works, the latter can be triggered by blocking the cluster state processing when a recover starts and only unblocking it shortly after the node receives the ShardActiveRequest. This is more reliable than using random cluster state processing delays because the random delays make it hard to reason about different timeouts that can be reached. closes #11989 --- .../store/IndicesStoreIntegrationTests.java | 89 +++++++++++++++---- 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java index 9a4783a2fa2..8162c951534 100644 --- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java +++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationTests.java @@ -20,27 +20,33 @@ package org.elasticsearch.indices.store; import com.google.common.base.Predicate; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.DiscoveryService; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.recovery.RecoverySource; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.test.disruption.SlowClusterStateProcessing; +import org.elasticsearch.test.disruption.BlockClusterStateProcessing; +import org.elasticsearch.test.disruption.SingleNodeDisruption; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.TransportModule; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; import org.junit.Test; import java.io.IOException; @@ -48,9 +54,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Future; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static java.lang.Thread.sleep; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -69,6 +76,7 @@ public class IndicesStoreIntegrationTests extends ElasticsearchIntegrationTest { // which is between 1 and 2 sec can cause each of the shard deletion requests to timeout. // to prevent this we are setting the timeout here to something highish ie. the default in practice .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT, new TimeValue(30, TimeUnit.SECONDS)) + .put(TransportModule.TRANSPORT_SERVICE_TYPE_KEY, MockTransportService.class.getName()) .build(); } @@ -79,7 +87,7 @@ public class IndicesStoreIntegrationTests extends ElasticsearchIntegrationTest { } @Test - @LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/11989") + @Slow public void indexCleanup() throws Exception { final String masterNode = internalCluster().startNode(Settings.builder().put("node.data", false)); final String node_1 = internalCluster().startNode(Settings.builder().put("node.master", false)); @@ -115,24 +123,30 @@ public class IndicesStoreIntegrationTests extends ElasticsearchIntegrationTest { assertThat(Files.exists(indexDirectory(node_3, "test")), equalTo(false)); logger.info("--> move shard from node_1 to node_3, and wait for relocation to finish"); - SlowClusterStateProcessing disruption = null; + if (randomBoolean()) { // sometimes add cluster-state delay to trigger observers in IndicesStore.ShardActiveRequestHandler - disruption = new SlowClusterStateProcessing(node_3, getRandom(), 0, 0, 1000, 2000); + SingleNodeDisruption disruption = new BlockClusterStateProcessing(node_3, getRandom()); internalCluster().setDisruptionScheme(disruption); + MockTransportService transportServiceNode3 = (MockTransportService) internalCluster().getInstance(TransportService.class, node_3); + CountDownLatch beginRelocationLatch = new CountDownLatch(1); + CountDownLatch endRelocationLatch = new CountDownLatch(1); + transportServiceNode3.addTracer(new ReclocationStartEndTracer(logger, beginRelocationLatch, endRelocationLatch)); + internalCluster().client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(new ShardId("test", 0), node_1, node_3)).get(); + // wait for relocation to start + beginRelocationLatch.await(); disruption.startDisrupting(); + // wait for relocation to finish + endRelocationLatch.await(); + // wait a little so that cluster state observer is registered + sleep(50); + disruption.stopDisrupting(); + } else { + internalCluster().client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(new ShardId("test", 0), node_1, node_3)).get(); } - internalCluster().client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(new ShardId("test", 0), node_1, node_3)).get(); clusterHealth = client().admin().cluster().prepareHealth() - .setWaitForNodes("4") .setWaitForRelocatingShards(0) .get(); assertThat(clusterHealth.isTimedOut(), equalTo(false)); - if (disruption != null) { - // we must stop the disruption here, else the delayed cluster state processing on the disrupted node - // can potentially delay registering the observer in IndicesStore.ShardActiveRequestHandler.messageReceived() - // and therefore sending the response for the shard active request for more than 10s - disruption.stopDisrupting(); - } assertThat(waitForShardDeletion(node_1, "test", 0), equalTo(false)); assertThat(waitForIndexDeletion(node_1, "test"), equalTo(false)); @@ -203,7 +217,8 @@ public class IndicesStoreIntegrationTests extends ElasticsearchIntegrationTest { assertThat(waitForShardDeletion(node_4, "test", 0), equalTo(false)); } - @Test @Slow + @Test + @Slow public void testShardActiveElseWhere() throws Exception { List nodes = internalCluster().startNodesAsync(2).get(); @@ -258,6 +273,7 @@ public class IndicesStoreIntegrationTests extends ElasticsearchIntegrationTest { .build(); } + @Override public boolean runOnlyOnMaster() { return false; } @@ -306,4 +322,45 @@ public class IndicesStoreIntegrationTests extends ElasticsearchIntegrationTest { }); return Files.exists(indexDirectory(server, index)); } + + /** + * This Tracer can be used to signal start and end of a recovery. + * This is used to test the following: + * Whenever a node deletes a shard because it was relocated somewhere else, it first + * checks if enough other copies are started somewhere else. The node sends a ShardActiveRequest + * to the other nodes that should have a copy according to cluster state. + * The nodes that receive this request check if the shard is in state STARTED in which case they + * respond with "true". If they have the shard in POST_RECOVERY they register a cluster state + * observer that checks at each update if the shard has moved to STARTED. + * To test that this mechanism actually works, this can be triggered by blocking the cluster + * state processing when a recover starts and only unblocking it shortly after the node receives + * the ShardActiveRequest. + */ + static class ReclocationStartEndTracer extends MockTransportService.Tracer { + private final ESLogger logger; + private final CountDownLatch beginRelocationLatch; + private final CountDownLatch receivedShardExistsRequestLatch; + + ReclocationStartEndTracer(ESLogger logger, CountDownLatch beginRelocationLatch, CountDownLatch receivedShardExistsRequestLatch) { + this.logger = logger; + this.beginRelocationLatch = beginRelocationLatch; + this.receivedShardExistsRequestLatch = receivedShardExistsRequestLatch; + } + + @Override + public void receivedRequest(long requestId, String action) { + if (action.equals(IndicesStore.ACTION_SHARD_EXISTS)) { + receivedShardExistsRequestLatch.countDown(); + logger.info("received: {}, relocation done", action); + } + } + + @Override + public void requestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) { + if (action.equals(RecoverySource.Actions.START_RECOVERY)) { + logger.info("sent: {}, relocation starts", action); + beginRelocationLatch.countDown(); + } + } + } } From 2d19868dd5b0c72aa6705ef1886bb417548dc11b Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 23 Jul 2015 12:06:06 +0200 Subject: [PATCH 9/9] Test: await fix on IndicesShardStoreRequestTests.testCorruptedShards --- .../admin/indices/shards/IndicesShardStoreRequestTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestTests.java index 1364c68cd07..4189455e783 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestTests.java @@ -139,6 +139,7 @@ public class IndicesShardStoreRequestTests extends ElasticsearchIntegrationTest } @Test + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/12416") public void testCorruptedShards() throws Exception { String index = "test"; internalCluster().ensureAtLeastNumDataNodes(2);