From c43ede5416c9a58e46d61616c02d6460478e7d25 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Sun, 13 Sep 2015 18:19:41 -0400 Subject: [PATCH 01/33] Improve IndexingMemoryController a bit: - promptly push indexing buffer changes to IndexWriter, instead of waiting for next refresh/flush - don't wait for merges to finish before dropping a shards's indexing buffer to 512 KB - fix NPE if indices.memory.index_buffer_size is in node's settings with a bytes (not %) unit - add some more logger.debug --- .../index/engine/EngineConfig.java | 7 +- .../index/engine/InternalEngine.java | 8 +- .../elasticsearch/index/shard/IndexShard.java | 29 ++++-- .../memory/IndexingMemoryController.java | 90 +++++++++---------- .../index/engine/InternalEngineTests.java | 2 + .../index/engine/ShadowEngineTests.java | 3 + .../memory/IndexingMemoryControllerIT.java | 38 ++++++++ 7 files changed, 114 insertions(+), 63 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index 6e6b0cfda69..899fa5f162a 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -94,8 +94,11 @@ public final class EngineConfig { public static final String INDEX_GC_DELETES_SETTING = "index.gc_deletes"; /** - * Index setting to control the initial index buffer size. - * This setting is not realtime updateable. + * Index setting to control the initial index buffer size. NOTE: this setting is somewhat + * useless, since IndexingMemoryController will take over quickly and partition the + * indices.memory.index_buffer_size for this node across all shards. + * + *

This setting is not realtime updateable. */ public static final String INDEX_BUFFER_SIZE_SETTING = "index.buffer_size"; diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 45f7801831e..a994520b29e 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -666,7 +666,6 @@ public class InternalEngine extends Engine { // since it flushes the index as well (though, in terms of concurrency, we are allowed to do it) try (ReleasableLock lock = readLock.acquire()) { ensureOpen(); - updateIndexWriterSettings(); searcherManager.maybeRefreshBlocking(); } catch (AlreadyClosedException e) { ensureOpen(); @@ -736,7 +735,6 @@ public class InternalEngine extends Engine { */ try (ReleasableLock lock = readLock.acquire()) { ensureOpen(); - updateIndexWriterSettings(); if (flushLock.tryLock() == false) { // if we can't get the lock right away we block if needed otherwise barf if (waitIfOngoing) { @@ -954,7 +952,6 @@ public class InternalEngine extends Engine { } } - /** * Closes the engine without acquiring the write lock. This should only be * called while the write lock is hold or in a disaster condition ie. if the engine @@ -1168,8 +1165,6 @@ public class InternalEngine extends Engine { return indexWriter.getConfig(); } - - private final class EngineMergeScheduler extends ElasticsearchConcurrentMergeScheduler { private final AtomicInteger numMergesInFlight = new AtomicInteger(0); private final AtomicBoolean isThrottling = new AtomicBoolean(); @@ -1245,11 +1240,10 @@ public class InternalEngine extends Engine { public void onSettingsChanged() { mergeScheduler.refreshConfig(); + updateIndexWriterSettings(); } public MergeStats getMergeStats() { return mergeScheduler.stats(); } - - } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index bf463db4bc6..c07ec52e386 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -985,15 +985,23 @@ public class IndexShard extends AbstractIndexShardComponent { } public void updateBufferSize(ByteSizeValue shardIndexingBufferSize, ByteSizeValue shardTranslogBufferSize) { + Engine engine = engineUnsafe(); + if (engine == null) { + logger.debug("updateBufferSize: engine is closed; skipping"); + return; + } + final EngineConfig config = engineConfig; final ByteSizeValue preValue = config.getIndexingBufferSize(); + config.setIndexingBufferSize(shardIndexingBufferSize); + // update engine if it is already started. - if (preValue.bytes() != shardIndexingBufferSize.bytes() && engineUnsafe() != null) { - // its inactive, make sure we do a refresh / full IW flush in this case, since the memory - // changes only after a "data" change has happened to the writer - // the index writer lazily allocates memory and a refresh will clean it all up. - if (shardIndexingBufferSize == EngineConfig.INACTIVE_SHARD_INDEXING_BUFFER && preValue != EngineConfig.INACTIVE_SHARD_INDEXING_BUFFER) { + if (preValue.bytes() != shardIndexingBufferSize.bytes()) { + if (shardIndexingBufferSize == EngineConfig.INACTIVE_SHARD_INDEXING_BUFFER) { + // it's inactive: make sure we do a refresh / full IW flush in this case, since the memory + // changes only after a "data" change has happened to the writer + // the index writer lazily allocates memory and a refresh will clean it all up. logger.debug("updating index_buffer_size from [{}] to (inactive) [{}]", preValue, shardIndexingBufferSize); try { refresh("update index buffer"); @@ -1003,11 +1011,12 @@ public class IndexShard extends AbstractIndexShardComponent { } else { logger.debug("updating index_buffer_size from [{}] to [{}]", preValue, shardIndexingBufferSize); } + + // so we push changes these changes down to IndexWriter: + engine.onSettingsChanged(); } - Engine engine = engineUnsafe(); - if (engine != null) { - engine.getTranslog().updateBuffer(shardTranslogBufferSize); - } + + engine.getTranslog().updateBuffer(shardTranslogBufferSize); } public void markAsInactive() { @@ -1129,6 +1138,8 @@ public class IndexShard extends AbstractIndexShardComponent { searchService.onRefreshSettings(settings); indexingService.onRefreshSettings(settings); if (change) { + engine().onSettingsChanged(); + // TODO: why force a refresh here...? refresh("apply settings"); } } diff --git a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java index 67d3e8eadfc..6b28d835191 100644 --- a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java +++ b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java @@ -51,6 +51,8 @@ import java.util.concurrent.ScheduledFuture; */ public class IndexingMemoryController extends AbstractLifecycleComponent { + public static final String INDEX_BUFFER_SIZE_SETTING = "indices.memory.index_buffer_size"; + private final ThreadPool threadPool; private final IndicesService indicesService; @@ -77,7 +79,7 @@ public class IndexingMemoryController extends AbstractLifecycleComponent shardsIndicesStatus = new HashMap<>(); - @Override public void run() { - EnumSet changes = EnumSet.noneOf(ShardStatusChangeType.class); - - changes.addAll(purgeDeletedAndClosedShards()); + EnumSet changes = purgeDeletedAndClosedShards(); final List activeToInactiveIndexingShards = new ArrayList<>(); final int activeShards = updateShardStatuses(changes, activeToInactiveIndexingShards); @@ -170,11 +170,15 @@ public class IndexingMemoryController extends AbstractLifecycleComponent inactiveTime.millis() && indexShard.mergeStats().getCurrent() == 0) { - // inactive for this amount of time, mark it - activeToInactiveIndexingShards.add(indexShard); - status.activeIndexing = false; - changes.add(ShardStatusChangeType.BECAME_INACTIVE); - logger.debug("marking shard [{}][{}] as inactive (inactive_time[{}]) indexing wise, setting size to [{}]", indexShard.shardId().index().name(), indexShard.shardId().id(), inactiveTime, EngineConfig.INACTIVE_SHARD_INDEXING_BUFFER); - } + // mark it as inactive only if enough time has passed + if (status.activeIndexing && (timeMS - status.timeMS) > inactiveTime.millis()) { + // inactive for this amount of time, mark it + activeToInactiveIndexingShards.add(indexShard); + status.activeIndexing = false; + changes.add(ShardStatusChangeType.BECAME_INACTIVE); + logger.debug("marking shard [{}][{}] as inactive (inactive_time[{}]) indexing wise, setting size to [{}]", + indexShard.shardId().index().name(), indexShard.shardId().id(), + inactiveTime, EngineConfig.INACTIVE_SHARD_INDEXING_BUFFER); } } else { if (!status.activeIndexing) { @@ -236,10 +242,9 @@ public class IndexingMemoryController extends AbstractLifecycleComponent getIWBufferSize("test3") == 16*1024*1024) == false) { + fail("failed to update shard indexing buffer size for test3 index to 16 MB; got: " + getIWBufferSize("test3")); + } + if (awaitBusy(() -> getIWBufferSize("test4") == 16*1024*1024) == false) { + fail("failed to update shard indexing buffer size for test4 index to 16 MB; got: " + getIWBufferSize("test4")); + } + } + + @Test + public void testIndexBufferNotPercent() throws InterruptedException { + // #13487: Make sure you can specify non-percent sized index buffer and not hit NPE + createNode(Settings.builder().put(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "32mb").build()); } private void createNode(Settings settings) { From 9148382be0822a4301aea95564b128c3351df7f4 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Mon, 14 Sep 2015 05:12:06 -0400 Subject: [PATCH 02/33] even if engineUnsafe() is null we still must update the EngineConfig --- .../elasticsearch/index/shard/IndexShard.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index c07ec52e386..1b0bbe92bb3 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -115,9 +115,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -/** - * - */ public class IndexShard extends AbstractIndexShardComponent { private final ThreadPool threadPool; @@ -985,17 +982,18 @@ public class IndexShard extends AbstractIndexShardComponent { } public void updateBufferSize(ByteSizeValue shardIndexingBufferSize, ByteSizeValue shardTranslogBufferSize) { - Engine engine = engineUnsafe(); - if (engine == null) { - logger.debug("updateBufferSize: engine is closed; skipping"); - return; - } final EngineConfig config = engineConfig; final ByteSizeValue preValue = config.getIndexingBufferSize(); config.setIndexingBufferSize(shardIndexingBufferSize); + Engine engine = engineUnsafe(); + if (engine == null) { + logger.debug("updateBufferSize: engine is closed; skipping"); + return; + } + // update engine if it is already started. if (preValue.bytes() != shardIndexingBufferSize.bytes()) { if (shardIndexingBufferSize == EngineConfig.INACTIVE_SHARD_INDEXING_BUFFER) { @@ -1278,6 +1276,8 @@ public class IndexShard extends AbstractIndexShardComponent { return engine; } + /** NOTE: returns null if engine is not yet started (e.g. recovery phase 1, copying over index files, is still running), or if engine is + * closed. */ protected Engine engineUnsafe() { return this.currentEngineReference.get(); } From 274044e7dd7c8964a5551e8f6b0aa44286f4b6ba Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Mon, 14 Sep 2015 05:18:07 -0400 Subject: [PATCH 03/33] push settings change before refresh, when shard becomes inactive --- .../main/java/org/elasticsearch/index/shard/IndexShard.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 1b0bbe92bb3..4d033997d46 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -996,6 +996,9 @@ public class IndexShard extends AbstractIndexShardComponent { // update engine if it is already started. if (preValue.bytes() != shardIndexingBufferSize.bytes()) { + // so we push changes these changes down to IndexWriter: + engine.onSettingsChanged(); + if (shardIndexingBufferSize == EngineConfig.INACTIVE_SHARD_INDEXING_BUFFER) { // it's inactive: make sure we do a refresh / full IW flush in this case, since the memory // changes only after a "data" change has happened to the writer @@ -1009,9 +1012,6 @@ public class IndexShard extends AbstractIndexShardComponent { } else { logger.debug("updating index_buffer_size from [{}] to [{}]", preValue, shardIndexingBufferSize); } - - // so we push changes these changes down to IndexWriter: - engine.onSettingsChanged(); } engine.getTranslog().updateBuffer(shardTranslogBufferSize); From 8dc08f70b4553f028c300ea0b617448b2369a036 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Mon, 14 Sep 2015 05:32:59 -0400 Subject: [PATCH 04/33] remove forced refresh on settings update --- .../java/org/elasticsearch/index/engine/InternalEngine.java | 4 ++++ .../main/java/org/elasticsearch/index/shard/IndexShard.java | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index a994520b29e..f4a77cfc610 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -1241,6 +1241,10 @@ public class InternalEngine extends Engine { public void onSettingsChanged() { mergeScheduler.refreshConfig(); updateIndexWriterSettings(); + // config().getVersionMapSize() may have changed: + checkVersionMapRefresh(); + // config().isEnableGcDeletes() or config.getGcDeletesInMillis() may have changed: + maybePruneDeletedTombstones(); } public MergeStats getMergeStats() { diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 4d033997d46..133f0991ef8 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -1137,8 +1137,6 @@ public class IndexShard extends AbstractIndexShardComponent { indexingService.onRefreshSettings(settings); if (change) { engine().onSettingsChanged(); - // TODO: why force a refresh here...? - refresh("apply settings"); } } } From 698da295732aad1afbac74545bbc66b4fa31142a Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Mon, 14 Sep 2015 05:58:21 -0400 Subject: [PATCH 05/33] pull out string constants for settings, and add javadocs --- .../memory/IndexingMemoryController.java | 65 +++++++++++++++---- .../memory/IndexingMemoryControllerIT.java | 6 +- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java index 6b28d835191..d52f06b13dd 100644 --- a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java +++ b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java @@ -51,7 +51,41 @@ import java.util.concurrent.ScheduledFuture; */ public class IndexingMemoryController extends AbstractLifecycleComponent { + /** How much heap (% or bytes) we will share across all actively indexing shards on this node (default: 10%). */ public static final String INDEX_BUFFER_SIZE_SETTING = "indices.memory.index_buffer_size"; + + /** Only applies when indices.memory.index_buffer_size is a %, to set a floor on the actual size in bytes (default: 48 MB). */ + public static final String MIN_INDEX_BUFFER_SIZE_SETTING = "indices.memory.min_index_buffer_size"; + + /** Only applies when indices.memory.index_buffer_size is a %, to set a ceiling on the actual size in bytes (default: not set). */ + public static final String MAX_INDEX_BUFFER_SIZE_SETTING = "indices.memory.max_index_buffer_size"; + + /** Sets a floor on the per-shard index buffer size (default: 4 MB). */ + public static final String MIN_SHARD_INDEX_BUFFER_SIZE_SETTING = "indices.memory.min_shard_index_buffer_size"; + + /** Sets a ceiling on the per-shard index buffer size (default: 512 MB). */ + public static final String MAX_SHARD_INDEX_BUFFER_SIZE_SETTING = "indices.memory.max_shard_index_buffer_size"; + + /** How much heap (% or bytes) we will share across all actively indexing shards for the translog buffer (default: 1%). */ + public static final String TRANSLOG_BUFFER_SIZE_SETTING = "indices.memory.translog_buffer_size"; + + /** Only applies when indices.memory.translog_buffer_size is a %, to set a floor on the actual size in bytes (default: 256 KB). */ + public static final String MIN_TRANSLOG_BUFFER_SIZE_SETTING = "indices.memory.min_translog_buffer_size"; + + /** Only applies when indices.memory.translog_buffer_size is a %, to set a ceiling on the actual size in bytes (default: not set). */ + public static final String MAX_TRANSLOG_BUFFER_SIZE_SETTING = "indices.memory.max_translog_buffer_size"; + + /** Sets a floor on the per-shard translog buffer size (default: 2 KB). */ + public static final String MIN_SHARD_TRANSLOG_BUFFER_SIZE_SETTING = "indices.memory.min_shard_translog_buffer_size"; + + /** Sets a ceiling on the per-shard translog buffer size (default: 64 KB). */ + public static final String MAX_SHARD_TRANSLOG_BUFFER_SIZE_SETTING = "indices.memory.max_shard_translog_buffer_size"; + + /** If we see no indexing operations after this much time for a given shard, we consider that shard inactive (default: 5 minutes). */ + public static final String SHARD_INACTIVE_TIME_SETTING = "indices.memory.shard_inactive_time"; + + /** How frequently we check shards to find inactive ones (default: 30 seconds). */ + public static final String SHARD_INACTIVE_INTERVAL_TIME_SETTING = "indices.memory.interval"; private final ThreadPool threadPool; private final IndicesService indicesService; @@ -83,8 +117,8 @@ public class IndexingMemoryController extends AbstractLifecycleComponent Date: Mon, 14 Sep 2015 08:41:31 -0400 Subject: [PATCH 06/33] logger.trace not logger.debug; fix test to verify bytes-sized index buffer size in node settings on node init took --- .../indices/memory/IndexingMemoryController.java | 4 ++-- .../indices/memory/IndexingMemoryControllerIT.java | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java index d52f06b13dd..2bf6946bc32 100644 --- a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java +++ b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java @@ -207,10 +207,10 @@ public class IndexingMemoryController extends AbstractLifecycleComponent Date: Mon, 14 Sep 2015 16:52:32 -0400 Subject: [PATCH 07/33] remove index.buffer_size setting --- .../cluster/metadata/MetaDataIndexUpgradeService.java | 2 -- .../org/elasticsearch/index/engine/EngineConfig.java | 10 ++-------- .../index/engine/InternalEngineTests.java | 9 +++++---- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java index 2d26ebfd941..bc033fcd6d6 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java @@ -26,7 +26,6 @@ import org.elasticsearch.cluster.routing.DjbHashFunction; import org.elasticsearch.cluster.routing.HashFunction; import org.elasticsearch.cluster.routing.SimpleHashFunction; import org.elasticsearch.cluster.routing.UnassignedInfo; -import org.elasticsearch.common.Classes; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -219,7 +218,6 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { /** All known byte-sized settings for an index. */ public static final Set INDEX_BYTES_SIZE_SETTINGS = ImmutableSet.of( - "index.buffer_size", "index.merge.policy.floor_segment", "index.merge.policy.max_merged_segment", "index.merge.policy.max_merge_size", diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index 6e6b0cfda69..d18097953aa 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -93,12 +93,6 @@ public final class EngineConfig { */ public static final String INDEX_GC_DELETES_SETTING = "index.gc_deletes"; - /** - * Index setting to control the initial index buffer size. - * This setting is not realtime updateable. - */ - public static final String INDEX_BUFFER_SIZE_SETTING = "index.buffer_size"; - /** * Index setting to change the low level lucene codec used for writing new segments. * This setting is not realtime updateable. @@ -152,7 +146,7 @@ public final class EngineConfig { this.optimizeAutoGenerateId = indexSettings.getAsBoolean(EngineConfig.INDEX_OPTIMIZE_AUTOGENERATED_ID_SETTING, false); this.compoundOnFlush = indexSettings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, compoundOnFlush); codecName = indexSettings.get(EngineConfig.INDEX_CODEC_SETTING, EngineConfig.DEFAULT_CODEC_NAME); - indexingBufferSize = indexSettings.getAsBytesSize(INDEX_BUFFER_SIZE_SETTING, DEFAULT_INDEX_BUFFER_SIZE); + indexingBufferSize = DEFAULT_INDEX_BUFFER_SIZE; gcDeletesInMillis = indexSettings.getAsTime(INDEX_GC_DELETES_SETTING, EngineConfig.DEFAULT_GC_DELETES).millis(); versionMapSizeSetting = indexSettings.get(INDEX_VERSION_MAP_SIZE, DEFAULT_VERSION_MAP_SIZE); updateVersionMapSize(); @@ -167,7 +161,7 @@ public final class EngineConfig { private void updateVersionMapSize() { if (versionMapSizeSetting.endsWith("%")) { double percent = Double.parseDouble(versionMapSizeSetting.substring(0, versionMapSizeSetting.length() - 1)); - versionMapSize = new ByteSizeValue((long) (((double) indexingBufferSize.bytes() * (percent / 100)))); + versionMapSize = new ByteSizeValue((long) ((double) indexingBufferSize.bytes() * (percent / 100))); } else { versionMapSize = ByteSizeValue.parseBytesSizeValue(versionMapSizeSetting, INDEX_VERSION_MAP_SIZE); } diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 8e1c81af368..a1b577e5c6f 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.engine; import com.google.common.collect.ImmutableMap; + import org.apache.log4j.AppenderSkeleton; import org.apache.log4j.Level; import org.apache.log4j.LogManager; @@ -52,6 +53,8 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.Index; import org.elasticsearch.index.VersionType; @@ -1635,12 +1638,10 @@ public class InternalEngineTests extends ESTestCase { // #10312 @Test public void testDeletesAloneCanTriggerRefresh() throws Exception { - // Tiny indexing buffer: - Settings indexSettings = Settings.builder().put(defaultSettings) - .put(EngineConfig.INDEX_BUFFER_SIZE_SETTING, "1kb").build(); try (Store store = createStore(); - Engine engine = new InternalEngine(config(indexSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), newMergePolicy()), + Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), newMergePolicy()), false)) { + engine.config().setIndexingBufferSize(new ByteSizeValue(1, ByteSizeUnit.KB)); for (int i = 0; i < 100; i++) { String id = Integer.toString(i); ParsedDocument doc = testParsedDocument(id, id, "test", null, -1, -1, testDocument(), B_1, null); From a38bcc5d629065cc724e7cb7edaabb5c2074dc25 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 10 Sep 2015 17:17:10 +0200 Subject: [PATCH 08/33] [test] plugins simple RestIT tests don't work from IDE When running a RestIT test from the IDE, you actually start an internal node which does not automatically load the plugin you would like to test. We need to add: ```java @Override protected Collection> nodePlugins() { return pluginList(PLUGIN_HERE.class); } ``` Everything works fine when running from maven because each test basically: * installs elasticsearch * installs one plugin * starts elasticsearch with this plugin loaded * runs the test Note that this PR only fixes the fact we run an internal cluster with the expected plugin. Cloud tests will still fail when run from the IDE because is such a case you actually start an internal node with many mock plugins. And REST test suite for cloud plugins basically checks if the plugin is running by checking the output of NodesInfo API. And we check: ```yml - match: { nodes.$master.plugins.0.name: cloud-azure } - match: { nodes.$master.plugins.0.jvm: true } ``` But in that case, this condition is certainly false as we started also `mock-transport-service`, `mock-index-store`, `mock-engine-factory`, `node-mocks`, `asserting-local-transport`, `mock-search-service`. Closes #13479 --- .../elasticsearch/index/analysis/AnalysisICURestIT.java | 8 ++++++++ .../index/analysis/AnalysisKuromojiRestIT.java | 9 +++++++++ .../index/analysis/AnalysisPhoneticRestIT.java | 8 ++++++++ .../index/analysis/AnalysisSmartChineseRestIT.java | 8 ++++++++ .../index/analysis/AnalysisPolishRestIT.java | 8 ++++++++ .../org/elasticsearch/cloud/azure/CloudAzureRestIT.java | 8 ++++++++ .../java/org/elasticsearch/cloud/gce/CloudGCERestIT.java | 8 ++++++++ .../deletebyquery/test/rest/DeleteByQueryRestIT.java | 8 ++++++++ .../org/elasticsearch/cloud/aws/DiscoveryEc2RestIT.java | 8 ++++++++ .../discovery/multicast/MulticastDiscoveryRestIT.java | 7 +++++++ .../elasticsearch/plugin/example/JvmExampleRestIT.java | 7 +++++++ .../script/javascript/LangJavaScriptRestIT.java | 8 ++++++++ .../script/python/LangPythonScriptRestIT.java | 8 ++++++++ .../index/mapper/murmur3/MapperMurmur3RestIT.java | 8 ++++++++ .../index/mapper/size/MapperSizeRestIT.java | 8 ++++++++ .../repositories/s3/RepositoryS3RestIT.java | 8 ++++++++ 16 files changed, 127 insertions(+) diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICURestIT.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICURestIT.java index 8da56d5a72b..bfb2b96a5ba 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICURestIT.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICURestIT.java @@ -21,14 +21,22 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class AnalysisICURestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(AnalysisICUPlugin.class); + } + public AnalysisICURestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiRestIT.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiRestIT.java index ae51e491d6b..cb9435e430e 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiRestIT.java +++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiRestIT.java @@ -21,14 +21,23 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.analysis.kuromoji.AnalysisKuromojiPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class AnalysisKuromojiRestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(AnalysisKuromojiPlugin.class); + } + + public AnalysisKuromojiRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticRestIT.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticRestIT.java index 9d66bf24357..98380a07176 100644 --- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticRestIT.java +++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticRestIT.java @@ -21,14 +21,22 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.analysis.AnalysisPhoneticPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class AnalysisPhoneticRestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(AnalysisPhoneticPlugin.class); + } + public AnalysisPhoneticRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseRestIT.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseRestIT.java index 16113b2b7ac..2a76c21d353 100644 --- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseRestIT.java +++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseRestIT.java @@ -21,14 +21,22 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.analysis.smartcn.AnalysisSmartChinesePlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class AnalysisSmartChineseRestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(AnalysisSmartChinesePlugin.class); + } + public AnalysisSmartChineseRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishRestIT.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishRestIT.java index 330ad87af74..c99ff75aebf 100644 --- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishRestIT.java +++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishRestIT.java @@ -21,14 +21,22 @@ package org.elasticsearch.index.analysis; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.analysis.stempel.AnalysisStempelPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class AnalysisPolishRestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(AnalysisStempelPlugin.class); + } + public AnalysisPolishRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/cloud-azure/src/test/java/org/elasticsearch/cloud/azure/CloudAzureRestIT.java b/plugins/cloud-azure/src/test/java/org/elasticsearch/cloud/azure/CloudAzureRestIT.java index aac8f9dea21..be74381da6a 100644 --- a/plugins/cloud-azure/src/test/java/org/elasticsearch/cloud/azure/CloudAzureRestIT.java +++ b/plugins/cloud-azure/src/test/java/org/elasticsearch/cloud/azure/CloudAzureRestIT.java @@ -21,14 +21,22 @@ package org.elasticsearch.cloud.azure; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.cloud.azure.CloudAzurePlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class CloudAzureRestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(CloudAzurePlugin.class); + } + public CloudAzureRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/cloud-gce/src/test/java/org/elasticsearch/cloud/gce/CloudGCERestIT.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/cloud/gce/CloudGCERestIT.java index 644a2e4aca3..e20965565f9 100644 --- a/plugins/cloud-gce/src/test/java/org/elasticsearch/cloud/gce/CloudGCERestIT.java +++ b/plugins/cloud-gce/src/test/java/org/elasticsearch/cloud/gce/CloudGCERestIT.java @@ -21,14 +21,22 @@ package org.elasticsearch.cloud.gce; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.cloud.gce.CloudGcePlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class CloudGCERestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(CloudGcePlugin.class); + } + public CloudGCERestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestIT.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestIT.java index 9674d541354..038f117b83f 100644 --- a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestIT.java +++ b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestIT.java @@ -21,14 +21,22 @@ package org.elasticsearch.plugin.deletebyquery.test.rest; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.deletebyquery.DeleteByQueryPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class DeleteByQueryRestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(DeleteByQueryPlugin.class); + } + public DeleteByQueryRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/DiscoveryEc2RestIT.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/DiscoveryEc2RestIT.java index 24ccf82a3d8..57021a085e6 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/DiscoveryEc2RestIT.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/DiscoveryEc2RestIT.java @@ -21,14 +21,22 @@ package org.elasticsearch.cloud.aws; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class DiscoveryEc2RestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(Ec2DiscoveryPlugin.class); + } + public DiscoveryEc2RestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/discovery-multicast/src/test/java/org/elasticsearch/plugin/discovery/multicast/MulticastDiscoveryRestIT.java b/plugins/discovery-multicast/src/test/java/org/elasticsearch/plugin/discovery/multicast/MulticastDiscoveryRestIT.java index c6af20c011e..d75c038a500 100644 --- a/plugins/discovery-multicast/src/test/java/org/elasticsearch/plugin/discovery/multicast/MulticastDiscoveryRestIT.java +++ b/plugins/discovery-multicast/src/test/java/org/elasticsearch/plugin/discovery/multicast/MulticastDiscoveryRestIT.java @@ -21,14 +21,21 @@ package org.elasticsearch.plugin.discovery.multicast; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class MulticastDiscoveryRestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(MulticastDiscoveryPlugin.class); + } + public MulticastDiscoveryRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleRestIT.java b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleRestIT.java index 74573a79289..2e9039ed67f 100644 --- a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleRestIT.java +++ b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleRestIT.java @@ -21,14 +21,21 @@ package org.elasticsearch.plugin.example; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class JvmExampleRestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(JvmExamplePlugin.class); + } + public JvmExampleRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavaScriptRestIT.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavaScriptRestIT.java index 8039715c3d1..18f18372b90 100644 --- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavaScriptRestIT.java +++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/LangJavaScriptRestIT.java @@ -21,14 +21,22 @@ package org.elasticsearch.script.javascript; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.javascript.JavaScriptPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class LangJavaScriptRestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(JavaScriptPlugin.class); + } + public LangJavaScriptRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonScriptRestIT.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonScriptRestIT.java index ee0a707644f..d7d0ca662c8 100644 --- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonScriptRestIT.java +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/LangPythonScriptRestIT.java @@ -21,14 +21,22 @@ package org.elasticsearch.script.python; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.python.PythonPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class LangPythonScriptRestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(PythonPlugin.class); + } + public LangPythonScriptRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3RestIT.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3RestIT.java index f440a343bc6..5dfc48570e6 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3RestIT.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3RestIT.java @@ -22,14 +22,22 @@ package org.elasticsearch.index.mapper.murmur3; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.mapper.MapperMurmur3Plugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class MapperMurmur3RestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(MapperMurmur3Plugin.class); + } + public MapperMurmur3RestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeRestIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeRestIT.java index ec7e000808e..c267160eeb7 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeRestIT.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeRestIT.java @@ -22,14 +22,22 @@ package org.elasticsearch.index.mapper.size; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.mapper.MapperSizePlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class MapperSizeRestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(MapperSizePlugin.class); + } + public MapperSizeRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java index d8e436b50bb..8521780b1d8 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java @@ -21,14 +21,22 @@ package org.elasticsearch.repositories.s3; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; +import java.util.Collection; public class RepositoryS3RestIT extends ESRestTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(S3RepositoryPlugin.class); + } + public RepositoryS3RestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } From f5c408535d88201d64c76c4d16f566e52112993f Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 12 Sep 2015 22:16:18 -0400 Subject: [PATCH 09/33] Remove and forbid use of com.google.common.base.Preconditions This commit removes and now forbids all uses of com.google.common.base.Preconditions across the codebase. This is one of many steps in the eventual removal of Guava as a dependency. Relates #13224 --- .../cluster/metadata/IndexMetaData.java | 2 +- .../elasticsearch/common/Preconditions.java | 67 +++++++++++++++++++ .../org/elasticsearch/common/cli/CliTool.java | 3 +- .../common/collect/CopyOnWriteHashMap.java | 17 +---- .../common/inject/AbstractModule.java | 2 +- .../common/inject/ConfigurationException.java | 2 +- .../common/inject/ConstructorBindingImpl.java | 2 +- .../common/inject/CreationException.java | 2 +- .../common/inject/EncounterImpl.java | 2 +- .../common/inject/InjectorShell.java | 19 +----- .../org/elasticsearch/common/inject/Key.java | 2 +- .../common/inject/PrivateModule.java | 2 +- .../common/inject/ProvisionException.java | 2 +- .../common/inject/TypeLiteral.java | 2 +- .../assistedinject/FactoryProvider2.java | 14 +--- .../inject/assistedinject/Parameter.java | 2 +- .../common/inject/internal/MoreTypes.java | 2 +- .../inject/internal/PrivateElementsImpl.java | 12 +--- .../common/inject/matcher/Matchers.java | 2 +- .../common/inject/spi/Elements.java | 32 ++------- .../inject/spi/MembersInjectorLookup.java | 2 +- .../common/inject/spi/ProviderLookup.java | 2 +- .../elasticsearch/common/unit/Fuzziness.java | 4 +- .../elasticsearch/common/unit/SizeValue.java | 2 +- .../common/util/AbstractBigArray.java | 2 +- .../common/util/AbstractPagedHashMap.java | 2 +- .../elasticsearch/common/util/BigArrays.java | 2 +- .../common/util/BigByteArray.java | 2 +- .../common/util/BigDoubleArray.java | 3 +- .../common/util/BigFloatArray.java | 3 +- .../common/util/BigIntArray.java | 2 +- .../common/util/BigLongArray.java | 2 +- .../common/util/CollectionUtils.java | 19 +----- .../gateway/MetaDataStateFormat.java | 15 +---- .../plain/BinaryDVNumericIndexFieldData.java | 3 +- .../plain/PackedArrayIndexFieldData.java | 2 +- .../plain/SortedNumericDVIndexFieldData.java | 17 +---- .../cardinality/CardinalityAggregator.java | 3 +- .../cardinality/HyperLogLogPlusPlus.java | 4 +- .../resources/forbidden/all-signatures.txt | 4 +- 40 files changed, 121 insertions(+), 164 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/common/Preconditions.java diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index dc4691e6ee1..158d5818b29 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -21,7 +21,6 @@ package org.elasticsearch.cluster.metadata; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import com.google.common.base.Preconditions; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.Diff; @@ -34,6 +33,7 @@ import org.elasticsearch.cluster.routing.HashFunction; import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedXContent; diff --git a/core/src/main/java/org/elasticsearch/common/Preconditions.java b/core/src/main/java/org/elasticsearch/common/Preconditions.java new file mode 100644 index 00000000000..9cb6622d496 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/Preconditions.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common; + +import java.util.Locale; + +public class Preconditions { + private Preconditions() { + } + + public static void checkArgument(boolean expression) { + if (!expression) { + throw new IllegalArgumentException(); + } + } + + public static void checkArgument(boolean expression, @Nullable Object errorMessage) { + if (!expression) { + throw new IllegalArgumentException(String.valueOf(errorMessage)); + } + } + + public static void checkArgument(boolean expression, @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { + if (!expression) { + throw new IllegalArgumentException(format(errorMessageTemplate, errorMessageArgs)); + } + } + + public static void checkState(boolean expression) { + if (!expression) { + throw new IllegalStateException(); + } + } + + public static void checkState(boolean expression, @Nullable Object errorMessage) { + if (!expression) { + throw new IllegalStateException(String.valueOf(errorMessage)); + } + } + + public static void checkState(boolean expression, @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { + if (!expression) { + throw new IllegalStateException(format(errorMessageTemplate, errorMessageArgs)); + } + } + + private static String format(@Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { + return String.format(Locale.ROOT, errorMessageTemplate, errorMessageArgs); + } +} diff --git a/core/src/main/java/org/elasticsearch/common/cli/CliTool.java b/core/src/main/java/org/elasticsearch/common/cli/CliTool.java index d264232aba1..bea819151f9 100644 --- a/core/src/main/java/org/elasticsearch/common/cli/CliTool.java +++ b/core/src/main/java/org/elasticsearch/common/cli/CliTool.java @@ -19,11 +19,10 @@ package org.elasticsearch.common.cli; -import com.google.common.base.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; -import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.node.internal.InternalSettingsPreparer; diff --git a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java index 8ec1a1385a7..fd288e859e7 100644 --- a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java +++ b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java @@ -19,25 +19,12 @@ package org.elasticsearch.common.collect; -import com.google.common.base.Preconditions; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.google.common.collect.UnmodifiableIterator; import org.apache.lucene.util.mutable.MutableValueInt; +import org.elasticsearch.common.Preconditions; import java.lang.reflect.Array; -import java.util.AbstractMap; -import java.util.AbstractSet; -import java.util.ArrayDeque; -import java.util.Arrays; -import java.util.Collection; -import java.util.Deque; -import java.util.Iterator; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; +import java.util.*; import java.util.stream.Stream; /** diff --git a/core/src/main/java/org/elasticsearch/common/inject/AbstractModule.java b/core/src/main/java/org/elasticsearch/common/inject/AbstractModule.java index af908845509..a30af09c1bc 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/AbstractModule.java +++ b/core/src/main/java/org/elasticsearch/common/inject/AbstractModule.java @@ -27,7 +27,7 @@ import org.elasticsearch.common.inject.spi.TypeListener; import java.lang.annotation.Annotation; import java.util.Objects; -import static com.google.common.base.Preconditions.checkState; +import static org.elasticsearch.common.Preconditions.checkState; /** * A support class for {@link Module}s which reduces repetition and results in diff --git a/core/src/main/java/org/elasticsearch/common/inject/ConfigurationException.java b/core/src/main/java/org/elasticsearch/common/inject/ConfigurationException.java index db7f2507332..57ce045a09e 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/ConfigurationException.java +++ b/core/src/main/java/org/elasticsearch/common/inject/ConfigurationException.java @@ -22,7 +22,7 @@ import org.elasticsearch.common.inject.spi.Message; import java.util.Collection; -import static com.google.common.base.Preconditions.checkState; +import static org.elasticsearch.common.Preconditions.checkState; /** * Thrown when a programming error such as a misplaced annotation, illegal binding, or unsupported diff --git a/core/src/main/java/org/elasticsearch/common/inject/ConstructorBindingImpl.java b/core/src/main/java/org/elasticsearch/common/inject/ConstructorBindingImpl.java index 221b19b7ef6..8f5daf202b9 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/ConstructorBindingImpl.java +++ b/core/src/main/java/org/elasticsearch/common/inject/ConstructorBindingImpl.java @@ -25,7 +25,7 @@ import org.elasticsearch.common.inject.spi.InjectionPoint; import java.util.Set; -import static com.google.common.base.Preconditions.checkState; +import static org.elasticsearch.common.Preconditions.checkState; class ConstructorBindingImpl extends BindingImpl implements ConstructorBinding { diff --git a/core/src/main/java/org/elasticsearch/common/inject/CreationException.java b/core/src/main/java/org/elasticsearch/common/inject/CreationException.java index 4553bc2785f..05c5f04823e 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/CreationException.java +++ b/core/src/main/java/org/elasticsearch/common/inject/CreationException.java @@ -22,7 +22,7 @@ import org.elasticsearch.common.inject.spi.Message; import java.util.Collection; -import static com.google.common.base.Preconditions.checkArgument; +import static org.elasticsearch.common.Preconditions.checkArgument; /** * Thrown when errors occur while creating a {@link Injector}. Includes a list of encountered diff --git a/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java b/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java index 83b8e446d15..d1cf9fb3e8c 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java +++ b/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java @@ -25,7 +25,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static com.google.common.base.Preconditions.checkState; +import static org.elasticsearch.common.Preconditions.checkState; /** * @author jessewilson@google.com (Jesse Wilson) diff --git a/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java b/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java index 510d9b59c93..1822c903b07 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java +++ b/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java @@ -17,28 +17,15 @@ package org.elasticsearch.common.inject; import com.google.common.collect.ImmutableSet; -import org.elasticsearch.common.inject.internal.Errors; -import org.elasticsearch.common.inject.internal.ErrorsException; -import org.elasticsearch.common.inject.internal.InternalContext; -import org.elasticsearch.common.inject.internal.InternalFactory; -import org.elasticsearch.common.inject.internal.PrivateElementsImpl; -import org.elasticsearch.common.inject.internal.ProviderInstanceBindingImpl; -import org.elasticsearch.common.inject.internal.Scoping; -import org.elasticsearch.common.inject.internal.SourceProvider; -import org.elasticsearch.common.inject.internal.Stopwatch; -import org.elasticsearch.common.inject.spi.Dependency; -import org.elasticsearch.common.inject.spi.Element; -import org.elasticsearch.common.inject.spi.Elements; -import org.elasticsearch.common.inject.spi.InjectionPoint; -import org.elasticsearch.common.inject.spi.PrivateElements; -import org.elasticsearch.common.inject.spi.TypeListenerBinding; +import org.elasticsearch.common.inject.internal.*; +import org.elasticsearch.common.inject.spi.*; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.logging.Logger; -import static com.google.common.base.Preconditions.checkState; +import static org.elasticsearch.common.Preconditions.checkState; import static org.elasticsearch.common.inject.Scopes.SINGLETON; /** diff --git a/core/src/main/java/org/elasticsearch/common/inject/Key.java b/core/src/main/java/org/elasticsearch/common/inject/Key.java index e2bbcf840b6..8ee801ccea3 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/Key.java +++ b/core/src/main/java/org/elasticsearch/common/inject/Key.java @@ -24,7 +24,7 @@ import java.lang.annotation.Annotation; import java.lang.reflect.Type; import java.util.Objects; -import static com.google.common.base.Preconditions.checkArgument; +import static org.elasticsearch.common.Preconditions.checkArgument; /** * Binding key consisting of an injection type and an optional annotation. diff --git a/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java b/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java index caf6a23597f..4ecaaff17f3 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java +++ b/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java @@ -27,7 +27,7 @@ import org.elasticsearch.common.inject.spi.TypeListener; import java.lang.annotation.Annotation; -import static com.google.common.base.Preconditions.checkState; +import static org.elasticsearch.common.Preconditions.checkState; /** * A module whose configuration information is hidden from its environment by default. Only bindings diff --git a/core/src/main/java/org/elasticsearch/common/inject/ProvisionException.java b/core/src/main/java/org/elasticsearch/common/inject/ProvisionException.java index b124dfc5ad6..dea1bfba5ad 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/ProvisionException.java +++ b/core/src/main/java/org/elasticsearch/common/inject/ProvisionException.java @@ -23,7 +23,7 @@ import org.elasticsearch.common.inject.spi.Message; import java.util.Collection; import java.util.Collections; -import static com.google.common.base.Preconditions.checkArgument; +import static org.elasticsearch.common.Preconditions.checkArgument; /** * Indicates that there was a runtime failure while providing an instance. diff --git a/core/src/main/java/org/elasticsearch/common/inject/TypeLiteral.java b/core/src/main/java/org/elasticsearch/common/inject/TypeLiteral.java index de9101d747d..44c89e4a3c4 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/TypeLiteral.java +++ b/core/src/main/java/org/elasticsearch/common/inject/TypeLiteral.java @@ -24,7 +24,7 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; -import static com.google.common.base.Preconditions.checkArgument; +import static org.elasticsearch.common.Preconditions.checkArgument; import static org.elasticsearch.common.inject.internal.MoreTypes.canonicalize; /** diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java index 821d729cbaa..215a323be0a 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java +++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java @@ -18,17 +18,7 @@ package org.elasticsearch.common.inject.assistedinject; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.inject.Binder; -import org.elasticsearch.common.inject.Binding; -import org.elasticsearch.common.inject.ConfigurationException; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.Key; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.inject.Provider; -import org.elasticsearch.common.inject.ProvisionException; -import org.elasticsearch.common.inject.TypeLiteral; +import org.elasticsearch.common.inject.*; import org.elasticsearch.common.inject.internal.Errors; import org.elasticsearch.common.inject.internal.ErrorsException; import org.elasticsearch.common.inject.spi.Message; @@ -43,7 +33,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import static com.google.common.base.Preconditions.checkState; +import static org.elasticsearch.common.Preconditions.checkState; import static org.elasticsearch.common.inject.internal.Annotations.getKey; /** diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java index a882b9fb280..1872a419a76 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java +++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java @@ -22,7 +22,7 @@ import java.lang.annotation.Annotation; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; -import static com.google.common.base.Preconditions.checkArgument; +import static org.elasticsearch.common.Preconditions.checkArgument; /** * Models a method or constructor parameter. diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/MoreTypes.java b/core/src/main/java/org/elasticsearch/common/inject/internal/MoreTypes.java index 6eb6f404fae..f156a389501 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/internal/MoreTypes.java +++ b/core/src/main/java/org/elasticsearch/common/inject/internal/MoreTypes.java @@ -30,7 +30,7 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; -import static com.google.common.base.Preconditions.checkArgument; +import static org.elasticsearch.common.Preconditions.checkArgument; /** * Static methods for working with types that we aren't publishing in the diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java b/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java index 5d2bbad2b2c..d1e65ae6ce6 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java +++ b/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java @@ -25,16 +25,10 @@ import org.elasticsearch.common.inject.spi.Element; import org.elasticsearch.common.inject.spi.ElementVisitor; import org.elasticsearch.common.inject.spi.PrivateElements; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; +import static org.elasticsearch.common.Preconditions.checkArgument; +import static org.elasticsearch.common.Preconditions.checkState; /** * @author jessewilson@google.com (Jesse Wilson) diff --git a/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java b/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java index 7e663db2281..b502b28c61c 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java +++ b/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java @@ -24,7 +24,7 @@ import java.lang.reflect.AnnotatedElement; import java.lang.reflect.Method; import java.util.Objects; -import static com.google.common.base.Preconditions.checkArgument; +import static org.elasticsearch.common.Preconditions.checkArgument; /** * Matcher implementations. Supports matching classes and methods. diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java b/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java index 9c50a1aa68c..c8786d375cc 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java +++ b/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java @@ -16,43 +16,19 @@ package org.elasticsearch.common.inject.spi; -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.inject.Binder; -import org.elasticsearch.common.inject.Binding; -import org.elasticsearch.common.inject.Key; -import org.elasticsearch.common.inject.MembersInjector; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.inject.PrivateBinder; -import org.elasticsearch.common.inject.PrivateModule; -import org.elasticsearch.common.inject.Provider; -import org.elasticsearch.common.inject.Scope; -import org.elasticsearch.common.inject.Stage; -import org.elasticsearch.common.inject.TypeLiteral; +import org.elasticsearch.common.inject.*; import org.elasticsearch.common.inject.binder.AnnotatedBindingBuilder; import org.elasticsearch.common.inject.binder.AnnotatedConstantBindingBuilder; import org.elasticsearch.common.inject.binder.AnnotatedElementBuilder; -import org.elasticsearch.common.inject.internal.AbstractBindingBuilder; -import org.elasticsearch.common.inject.internal.BindingBuilder; -import org.elasticsearch.common.inject.internal.ConstantBindingBuilderImpl; -import org.elasticsearch.common.inject.internal.Errors; -import org.elasticsearch.common.inject.internal.ExposureBuilder; -import org.elasticsearch.common.inject.internal.PrivateElementsImpl; -import org.elasticsearch.common.inject.internal.ProviderMethodsModule; -import org.elasticsearch.common.inject.internal.SourceProvider; +import org.elasticsearch.common.inject.internal.*; import org.elasticsearch.common.inject.matcher.Matcher; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import java.lang.annotation.Annotation; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; -import static com.google.common.base.Preconditions.checkArgument; +import static org.elasticsearch.common.Preconditions.checkArgument; /** * Exposes elements of a module so they can be inspected, validated or {@link diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/MembersInjectorLookup.java b/core/src/main/java/org/elasticsearch/common/inject/spi/MembersInjectorLookup.java index f53d66265c2..89b04ce16ba 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/spi/MembersInjectorLookup.java +++ b/core/src/main/java/org/elasticsearch/common/inject/spi/MembersInjectorLookup.java @@ -22,7 +22,7 @@ import org.elasticsearch.common.inject.TypeLiteral; import java.util.Objects; -import static com.google.common.base.Preconditions.checkState; +import static org.elasticsearch.common.Preconditions.checkState; /** * A lookup of the members injector for a type. Lookups are created explicitly in a module using diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderLookup.java b/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderLookup.java index 61ae67a6e30..47c0a96cabb 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderLookup.java +++ b/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderLookup.java @@ -22,7 +22,7 @@ import org.elasticsearch.common.inject.Provider; import java.util.Objects; -import static com.google.common.base.Preconditions.checkState; +import static org.elasticsearch.common.Preconditions.checkState; /** * A lookup of the provider for a type. Lookups are created explicitly in a module using diff --git a/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java b/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java index cfcd209b435..60335ed49fb 100644 --- a/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java +++ b/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java @@ -18,10 +18,8 @@ */ package org.elasticsearch.common.unit; -import com.google.common.base.Preconditions; -import org.apache.lucene.search.FuzzyQuery; -import org.apache.lucene.util.automaton.LevenshteinAutomata; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; diff --git a/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java b/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java index 586f305df00..bc7d913d9d8 100644 --- a/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java +++ b/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java @@ -19,8 +19,8 @@ package org.elasticsearch.common.unit; -import com.google.common.base.Preconditions; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; diff --git a/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java b/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java index 1523b8058d2..2328c8bc65e 100644 --- a/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java @@ -19,10 +19,10 @@ package org.elasticsearch.common.util; -import com.google.common.base.Preconditions; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.cache.recycler.PageCacheRecycler; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.recycler.Recycler; diff --git a/core/src/main/java/org/elasticsearch/common/util/AbstractPagedHashMap.java b/core/src/main/java/org/elasticsearch/common/util/AbstractPagedHashMap.java index b48155776ed..7b1298beab8 100644 --- a/core/src/main/java/org/elasticsearch/common/util/AbstractPagedHashMap.java +++ b/core/src/main/java/org/elasticsearch/common/util/AbstractPagedHashMap.java @@ -20,7 +20,7 @@ package org.elasticsearch.common.util; import com.carrotsearch.hppc.BitMixer; -import com.google.common.base.Preconditions; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.lease.Releasable; /** diff --git a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java index 90db1bd73a0..49e338ef87b 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java @@ -19,12 +19,12 @@ package org.elasticsearch.common.util; -import com.google.common.base.Preconditions; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.cache.recycler.PageCacheRecycler; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.inject.Inject; diff --git a/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java b/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java index 2a445cc3941..7b7c69de9e3 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java @@ -19,10 +19,10 @@ package org.elasticsearch.common.util; -import com.google.common.base.Preconditions; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.Preconditions; import java.util.Arrays; diff --git a/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java b/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java index 38df32c83c0..e12138deafc 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java @@ -19,10 +19,9 @@ package org.elasticsearch.common.util; -import com.google.common.base.Preconditions; - import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.Preconditions; import java.util.Arrays; diff --git a/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java b/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java index fc73c72e6f2..a56ac31a149 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java @@ -19,10 +19,9 @@ package org.elasticsearch.common.util; -import com.google.common.base.Preconditions; - import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.Preconditions; import java.util.Arrays; diff --git a/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java b/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java index c07280b41a0..82bc451df29 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java @@ -19,9 +19,9 @@ package org.elasticsearch.common.util; -import com.google.common.base.Preconditions; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.Preconditions; import java.util.Arrays; diff --git a/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java b/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java index 286dc53b933..57750b3b2c9 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java @@ -19,9 +19,9 @@ package org.elasticsearch.common.util; -import com.google.common.base.Preconditions; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.Preconditions; import java.util.Arrays; diff --git a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java index 2791ea6e3ee..6f3d701a050 100644 --- a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java +++ b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java @@ -23,24 +23,11 @@ import com.carrotsearch.hppc.DoubleArrayList; import com.carrotsearch.hppc.FloatArrayList; import com.carrotsearch.hppc.LongArrayList; import com.carrotsearch.hppc.ObjectArrayList; -import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefArray; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.InPlaceMergeSorter; -import org.apache.lucene.util.IntroSorter; +import org.apache.lucene.util.*; +import org.elasticsearch.common.Preconditions; -import java.util.AbstractList; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.RandomAccess; +import java.util.*; /** Collections-related utility methods. */ public enum CollectionUtils { diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index b15a3273cc3..86a849b82cc 100644 --- a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -18,28 +18,19 @@ */ package org.elasticsearch.gateway; -import com.google.common.base.Preconditions; - import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.OutputStreamIndexOutput; -import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.*; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.store.IndexOutputOutputStream; import org.elasticsearch.common.lucene.store.InputStreamIndexInput; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.*; import java.io.FileNotFoundException; import java.io.IOException; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java index c78da7c6446..e00f7acde8d 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java @@ -19,8 +19,6 @@ package org.elasticsearch.index.fielddata.plain; -import com.google.common.base.Preconditions; - import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; @@ -29,6 +27,7 @@ import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.fielddata.AtomicNumericFieldData; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java index 89962093fb4..42cba5bf6d8 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.fielddata.plain; -import com.google.common.base.Preconditions; import org.apache.lucene.index.*; import org.apache.lucene.util.*; import org.apache.lucene.util.BitSet; @@ -27,6 +26,7 @@ import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java index 52686afc78d..f50047f21ef 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java @@ -19,24 +19,13 @@ package org.elasticsearch.index.fielddata.plain; -import com.google.common.base.Preconditions; - -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.*; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.index.Index; -import org.elasticsearch.index.fielddata.AtomicNumericFieldData; -import org.elasticsearch.index.fielddata.FieldData; -import org.elasticsearch.index.fielddata.FieldDataType; +import org.elasticsearch.index.fielddata.*; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.NumericDoubleValues; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource; import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java index 0a3918de46b..97b6a192c68 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java @@ -20,8 +20,6 @@ package org.elasticsearch.search.aggregations.metrics.cardinality; import com.carrotsearch.hppc.BitMixer; -import com.google.common.base.Preconditions; - import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedNumericDocValues; @@ -30,6 +28,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BigArrays; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java index 6eb5153e59d..3293f2372e7 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java @@ -19,12 +19,11 @@ package org.elasticsearch.search.aggregations.metrics.cardinality; -import com.google.common.base.Preconditions; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongBitSet; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.packed.PackedInts; -import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lease.Releasable; @@ -37,7 +36,6 @@ import org.elasticsearch.common.util.IntArray; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.Arrays; /** * Hyperloglog++ counter, implemented based on pseudo code from diff --git a/dev-tools/src/main/resources/forbidden/all-signatures.txt b/dev-tools/src/main/resources/forbidden/all-signatures.txt index cd87da1503d..d37cf2a35a5 100644 --- a/dev-tools/src/main/resources/forbidden/all-signatures.txt +++ b/dev-tools/src/main/resources/forbidden/all-signatures.txt @@ -97,9 +97,7 @@ com.google.common.base.Strings com.google.common.base.Throwables com.google.common.collect.Maps com.google.common.collect.Sets -com.google.common.base.Preconditions#checkNotNull(java.lang.Object) -com.google.common.base.Preconditions#checkNotNull(java.lang.Object, java.lang.Object) -com.google.common.base.Preconditions#checkNotNull(java.lang.Object, java.lang.String, java.lang.Object[]) +com.google.common.base.Preconditions com.google.common.collect.ImmutableSortedSet com.google.common.collect.Queues com.google.common.util.concurrent.ListenableFuture From 78bff93ab18da0042e5a9197fa6f709ef38ed5a9 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 13 Sep 2015 09:04:42 -0400 Subject: [PATCH 10/33] Completely remove Preconditions.checkState --- .../elasticsearch/common/Preconditions.java | 18 ----------- .../common/inject/AbstractModule.java | 7 ++--- .../common/inject/ConfigurationException.java | 9 +++--- .../common/inject/ConstructorBindingImpl.java | 18 +++++++---- .../common/inject/EncounterImpl.java | 30 +++++++++++++------ .../common/inject/InjectorShell.java | 13 +++++--- .../common/inject/PrivateModule.java | 6 ++-- .../assistedinject/FactoryProvider2.java | 6 ++-- .../inject/internal/PrivateElementsImpl.java | 5 ++-- .../inject/spi/MembersInjectorLookup.java | 11 +++---- .../common/inject/spi/ProviderLookup.java | 11 +++---- 11 files changed, 71 insertions(+), 63 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/Preconditions.java b/core/src/main/java/org/elasticsearch/common/Preconditions.java index 9cb6622d496..7eddef1f010 100644 --- a/core/src/main/java/org/elasticsearch/common/Preconditions.java +++ b/core/src/main/java/org/elasticsearch/common/Preconditions.java @@ -43,24 +43,6 @@ public class Preconditions { } } - public static void checkState(boolean expression) { - if (!expression) { - throw new IllegalStateException(); - } - } - - public static void checkState(boolean expression, @Nullable Object errorMessage) { - if (!expression) { - throw new IllegalStateException(String.valueOf(errorMessage)); - } - } - - public static void checkState(boolean expression, @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { - if (!expression) { - throw new IllegalStateException(format(errorMessageTemplate, errorMessageArgs)); - } - } - private static String format(@Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { return String.format(Locale.ROOT, errorMessageTemplate, errorMessageArgs); } diff --git a/core/src/main/java/org/elasticsearch/common/inject/AbstractModule.java b/core/src/main/java/org/elasticsearch/common/inject/AbstractModule.java index a30af09c1bc..86ef15875b8 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/AbstractModule.java +++ b/core/src/main/java/org/elasticsearch/common/inject/AbstractModule.java @@ -27,8 +27,6 @@ import org.elasticsearch.common.inject.spi.TypeListener; import java.lang.annotation.Annotation; import java.util.Objects; -import static org.elasticsearch.common.Preconditions.checkState; - /** * A support class for {@link Module}s which reduces repetition and results in * a more readable configuration. Simply extend this class, implement {@link @@ -54,8 +52,9 @@ public abstract class AbstractModule implements Module { @Override public final synchronized void configure(Binder builder) { - checkState(this.binder == null, "Re-entry is not allowed."); - + if (this.binder != null) { + throw new IllegalStateException("Re-entry is not allowed."); + } this.binder = Objects.requireNonNull(builder, "builder"); try { configure(); diff --git a/core/src/main/java/org/elasticsearch/common/inject/ConfigurationException.java b/core/src/main/java/org/elasticsearch/common/inject/ConfigurationException.java index 57ce045a09e..4afc6530621 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/ConfigurationException.java +++ b/core/src/main/java/org/elasticsearch/common/inject/ConfigurationException.java @@ -21,8 +21,7 @@ import org.elasticsearch.common.inject.internal.Errors; import org.elasticsearch.common.inject.spi.Message; import java.util.Collection; - -import static org.elasticsearch.common.Preconditions.checkState; +import java.util.Locale; /** * Thrown when a programming error such as a misplaced annotation, illegal binding, or unsupported @@ -48,8 +47,10 @@ public final class ConfigurationException extends RuntimeException { * Returns a copy of this configuration exception with the specified partial value. */ public ConfigurationException withPartialValue(Object partialValue) { - checkState(this.partialValue == null, - "Can't clobber existing partial value %s with %s", this.partialValue, partialValue); + if (this.partialValue != null) { + String message = String.format(Locale.ROOT, "Can't clobber existing partial value %s with %s", this.partialValue, partialValue); + throw new IllegalStateException(message); + } ConfigurationException result = new ConfigurationException(messages); result.partialValue = partialValue; return result; diff --git a/core/src/main/java/org/elasticsearch/common/inject/ConstructorBindingImpl.java b/core/src/main/java/org/elasticsearch/common/inject/ConstructorBindingImpl.java index 8f5daf202b9..0b5cec87d4b 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/ConstructorBindingImpl.java +++ b/core/src/main/java/org/elasticsearch/common/inject/ConstructorBindingImpl.java @@ -25,8 +25,6 @@ import org.elasticsearch.common.inject.spi.InjectionPoint; import java.util.Set; -import static org.elasticsearch.common.Preconditions.checkState; - class ConstructorBindingImpl extends BindingImpl implements ConstructorBinding { private final Factory factory; @@ -52,19 +50,25 @@ class ConstructorBindingImpl extends BindingImpl implements ConstructorBin @Override public V acceptTargetVisitor(BindingTargetVisitor visitor) { - checkState(factory.constructorInjector != null, "not initialized"); + if (factory.constructorInjector == null) { + throw new IllegalStateException("not initialized"); + } return visitor.visit(this); } @Override public InjectionPoint getConstructor() { - checkState(factory.constructorInjector != null, "Binding is not ready"); + if (factory.constructorInjector == null) { + throw new IllegalStateException("Binding is not ready"); + } return factory.constructorInjector.getConstructionProxy().getInjectionPoint(); } @Override public Set getInjectableMembers() { - checkState(factory.constructorInjector != null, "Binding is not ready"); + if (factory.constructorInjector == null) { + throw new IllegalStateException("Binding is not ready"); + } return factory.constructorInjector.getInjectableMembers(); } @@ -97,7 +101,9 @@ class ConstructorBindingImpl extends BindingImpl implements ConstructorBin @SuppressWarnings("unchecked") public T get(Errors errors, InternalContext context, Dependency dependency) throws ErrorsException { - checkState(constructorInjector != null, "Constructor not ready"); + if (constructorInjector == null) { + throw new IllegalStateException("Constructor not ready"); + } // This may not actually be safe because it could return a super type of T (if that's all the // client needs), but it should be OK in practice thanks to the wonders of erasure. diff --git a/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java b/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java index d1cf9fb3e8c..fc2bd484b41 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java +++ b/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java @@ -25,8 +25,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.elasticsearch.common.Preconditions.checkState; - /** * @author jessewilson@google.com (Jesse Wilson) */ @@ -61,7 +59,9 @@ final class EncounterImpl implements TypeEncounter { @Override public void register(MembersInjector membersInjector) { - checkState(valid, "Encounters may not be used after hear() returns."); + if (!valid) { + throw new IllegalStateException("Encounters may not be used after hear() returns."); + } if (membersInjectors == null) { membersInjectors = new ArrayList<>(); @@ -72,7 +72,9 @@ final class EncounterImpl implements TypeEncounter { @Override public void register(InjectionListener injectionListener) { - checkState(valid, "Encounters may not be used after hear() returns."); + if (!valid) { + throw new IllegalStateException("Encounters may not be used after hear() returns."); + } if (injectionListeners == null) { injectionListeners = new ArrayList<>(); @@ -83,25 +85,33 @@ final class EncounterImpl implements TypeEncounter { @Override public void addError(String message, Object... arguments) { - checkState(valid, "Encounters may not be used after hear() returns."); + if (!valid) { + throw new IllegalStateException("Encounters may not be used after hear() returns."); + } errors.addMessage(message, arguments); } @Override public void addError(Throwable t) { - checkState(valid, "Encounters may not be used after hear() returns."); + if (!valid) { + throw new IllegalStateException("Encounters may not be used after hear() returns."); + } errors.errorInUserCode(t, "An exception was caught and reported. Message: %s", t.getMessage()); } @Override public void addError(Message message) { - checkState(valid, "Encounters may not be used after hear() returns."); + if (!valid) { + throw new IllegalStateException("Encounters may not be used after hear() returns."); + } errors.addMessage(message); } @Override public Provider getProvider(Key key) { - checkState(valid, "Encounters may not be used after hear() returns."); + if (!valid) { + throw new IllegalStateException("Encounters may not be used after hear() returns."); + } return lookups.getProvider(key); } @@ -112,7 +122,9 @@ final class EncounterImpl implements TypeEncounter { @Override public MembersInjector getMembersInjector(TypeLiteral typeLiteral) { - checkState(valid, "Encounters may not be used after hear() returns."); + if (!valid) { + throw new IllegalStateException("Encounters may not be used after hear() returns."); + } return lookups.getMembersInjector(typeLiteral); } diff --git a/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java b/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java index 1822c903b07..8acc7d7c721 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java +++ b/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Objects; import java.util.logging.Logger; -import static org.elasticsearch.common.Preconditions.checkState; import static org.elasticsearch.common.inject.Scopes.SINGLETON; /** @@ -112,9 +111,15 @@ class InjectorShell { */ List build(Initializer initializer, BindingProcessor bindingProcessor, Stopwatch stopwatch, Errors errors) { - checkState(stage != null, "Stage not initialized"); - checkState(privateElements == null || parent != null, "PrivateElements with no parent"); - checkState(state != null, "no state. Did you remember to lock() ?"); + if (stage == null) { + throw new IllegalStateException("Stage not initialized"); + } + if (privateElements != null && parent == null) { + throw new IllegalStateException("PrivateElements with no parent"); + } + if (state == null) { + throw new IllegalStateException("no state. Did you remember to lock() ?"); + } InjectorImpl injector = new InjectorImpl(parent, state, initializer); if (privateElements != null) { diff --git a/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java b/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java index 4ecaaff17f3..a69ba75e0de 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java +++ b/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java @@ -27,8 +27,6 @@ import org.elasticsearch.common.inject.spi.TypeListener; import java.lang.annotation.Annotation; -import static org.elasticsearch.common.Preconditions.checkState; - /** * A module whose configuration information is hidden from its environment by default. Only bindings * that are explicitly exposed will be available to other modules and to the users of the injector. @@ -93,7 +91,9 @@ public abstract class PrivateModule implements Module { @Override public final synchronized void configure(Binder binder) { - checkState(this.binder == null, "Re-entry is not allowed."); + if (this.binder != null) { + throw new IllegalStateException("Re-entry is not allowed."); + } // Guice treats PrivateModules specially and passes in a PrivateBinder automatically. this.binder = (PrivateBinder) binder.skipSources(PrivateModule.class); diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java index 215a323be0a..b1d6b400831 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java +++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java @@ -33,7 +33,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.elasticsearch.common.Preconditions.checkState; import static org.elasticsearch.common.inject.internal.Annotations.getKey; /** @@ -182,8 +181,9 @@ public final class FactoryProvider2 implements InvocationHandler, Provider * Creates a child injector that binds the args, and returns the binding for the method's result. */ public Binding getBindingFromNewInjector(final Method method, final Object[] args) { - checkState(injector != null, - "Factories.create() factories cannot be used until they're initialized by Guice."); + if (injector == null) { + throw new IllegalStateException("Factories.create() factories cannot be used until they're initialized by Guice."); + } final Key returnType = returnTypesByMethod.get(method); diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java b/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java index d1e65ae6ce6..125c4d367af 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java +++ b/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.inject.spi.PrivateElements; import java.util.*; import static org.elasticsearch.common.Preconditions.checkArgument; -import static org.elasticsearch.common.Preconditions.checkState; /** * @author jessewilson@google.com (Jesse Wilson) @@ -82,7 +81,9 @@ public final class PrivateElementsImpl implements PrivateElements { } public void initInjector(Injector injector) { - checkState(this.injector == null, "injector already initialized"); + if (this.injector != null) { + throw new IllegalStateException("injector already initialized"); + } this.injector = Objects.requireNonNull(injector, "injector"); } diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/MembersInjectorLookup.java b/core/src/main/java/org/elasticsearch/common/inject/spi/MembersInjectorLookup.java index 89b04ce16ba..9bde6247359 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/spi/MembersInjectorLookup.java +++ b/core/src/main/java/org/elasticsearch/common/inject/spi/MembersInjectorLookup.java @@ -22,8 +22,6 @@ import org.elasticsearch.common.inject.TypeLiteral; import java.util.Objects; -import static org.elasticsearch.common.Preconditions.checkState; - /** * A lookup of the members injector for a type. Lookups are created explicitly in a module using * {@link org.elasticsearch.common.inject.Binder#getMembersInjector(Class) getMembersInjector()} statements: @@ -68,7 +66,9 @@ public final class MembersInjectorLookup implements Element { * @throws IllegalStateException if the delegate is already set */ public void initializeDelegate(MembersInjector delegate) { - checkState(this.delegate == null, "delegate already initialized"); + if (this.delegate != null) { + throw new IllegalStateException("delegate already initialized"); + } this.delegate = Objects.requireNonNull(delegate, "delegate"); } @@ -95,8 +95,9 @@ public final class MembersInjectorLookup implements Element { return new MembersInjector() { @Override public void injectMembers(T instance) { - checkState(delegate != null, - "This MembersInjector cannot be used until the Injector has been created."); + if (delegate == null) { + throw new IllegalStateException("This MembersInjector cannot be used until the Injector has been created."); + } delegate.injectMembers(instance); } diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderLookup.java b/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderLookup.java index 47c0a96cabb..464b8501af9 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderLookup.java +++ b/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderLookup.java @@ -22,8 +22,6 @@ import org.elasticsearch.common.inject.Provider; import java.util.Objects; -import static org.elasticsearch.common.Preconditions.checkState; - /** * A lookup of the provider for a type. Lookups are created explicitly in a module using * {@link org.elasticsearch.common.inject.Binder#getProvider(Class) getProvider()} statements: @@ -46,8 +44,9 @@ public final class ProviderLookup implements Element { @Override public T get() { - checkState(lookup.delegate != null, - "This Provider cannot be used until the Injector has been created."); + if (lookup.delegate == null) { + throw new IllegalStateException( "This Provider cannot be used until the Injector has been created."); + } return lookup.delegate.get(); } @@ -89,7 +88,9 @@ public final class ProviderLookup implements Element { * @throws IllegalStateException if the delegate is already set */ public void initializeDelegate(Provider delegate) { - checkState(this.delegate == null, "delegate already initialized"); + if (this.delegate != null) { + throw new IllegalStateException("delegate already initialized"); + } this.delegate = Objects.requireNonNull(delegate, "delegate"); } From acbede48d71a9260785e55e62fe99696ae356efa Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 13 Sep 2015 20:55:03 -0400 Subject: [PATCH 11/33] Completely remove Preconditions.checkArgument --- .../cluster/metadata/IndexMetaData.java | 9 ++-- .../elasticsearch/common/Preconditions.java | 49 ------------------- .../org/elasticsearch/common/cli/CliTool.java | 5 +- .../common/collect/CopyOnWriteHashMap.java | 17 +++++-- .../common/inject/CreationException.java | 6 +-- .../org/elasticsearch/common/inject/Key.java | 18 ++++--- .../common/inject/ProvisionException.java | 6 +-- .../common/inject/TypeLiteral.java | 39 +++++++++------ .../inject/assistedinject/Parameter.java | 7 ++- .../common/inject/internal/MoreTypes.java | 38 ++++++++------ .../inject/internal/PrivateElementsImpl.java | 6 +-- .../common/inject/matcher/Matchers.java | 7 ++- .../common/inject/spi/Elements.java | 6 +-- .../elasticsearch/common/unit/Fuzziness.java | 5 +- .../elasticsearch/common/unit/SizeValue.java | 5 +- .../common/util/AbstractBigArray.java | 13 +++-- .../common/util/AbstractPagedHashMap.java | 9 ++-- .../elasticsearch/common/util/BigArrays.java | 13 +++-- .../common/util/BigByteArray.java | 5 +- .../common/util/BigDoubleArray.java | 5 +- .../common/util/BigFloatArray.java | 5 +- .../common/util/BigIntArray.java | 5 +- .../common/util/BigLongArray.java | 5 +- .../common/util/CollectionUtils.java | 9 ++-- .../gateway/MetaDataStateFormat.java | 9 ++-- .../plain/BinaryDVNumericIndexFieldData.java | 5 +- .../plain/PackedArrayIndexFieldData.java | 5 +- .../plain/SortedNumericDVIndexFieldData.java | 5 +- .../cardinality/CardinalityAggregator.java | 5 +- .../cardinality/HyperLogLogPlusPlus.java | 13 +++-- 30 files changed, 175 insertions(+), 159 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/common/Preconditions.java diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 158d5818b29..2e2458cab60 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -33,7 +33,6 @@ import org.elasticsearch.cluster.routing.HashFunction; import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedXContent; @@ -205,8 +204,12 @@ public class IndexMetaData implements Diffable, FromXContentBuild private final boolean useTypeForRouting; private IndexMetaData(String index, long version, State state, Settings settings, ImmutableOpenMap mappings, ImmutableOpenMap aliases, ImmutableOpenMap customs) { - Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null) != null, "must specify numberOfShards for index [" + index + "]"); - Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, null) != null, "must specify numberOfReplicas for index [" + index + "]"); + if (settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null) == null) { + throw new IllegalArgumentException("must specify numberOfShards for index [" + index + "]"); + } + if (settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, null) == null) { + throw new IllegalArgumentException("must specify numberOfReplicas for index [" + index + "]"); + } this.index = index; this.version = version; this.state = state; diff --git a/core/src/main/java/org/elasticsearch/common/Preconditions.java b/core/src/main/java/org/elasticsearch/common/Preconditions.java deleted file mode 100644 index 7eddef1f010..00000000000 --- a/core/src/main/java/org/elasticsearch/common/Preconditions.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common; - -import java.util.Locale; - -public class Preconditions { - private Preconditions() { - } - - public static void checkArgument(boolean expression) { - if (!expression) { - throw new IllegalArgumentException(); - } - } - - public static void checkArgument(boolean expression, @Nullable Object errorMessage) { - if (!expression) { - throw new IllegalArgumentException(String.valueOf(errorMessage)); - } - } - - public static void checkArgument(boolean expression, @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { - if (!expression) { - throw new IllegalArgumentException(format(errorMessageTemplate, errorMessageArgs)); - } - } - - private static String format(@Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { - return String.format(Locale.ROOT, errorMessageTemplate, errorMessageArgs); - } -} diff --git a/core/src/main/java/org/elasticsearch/common/cli/CliTool.java b/core/src/main/java/org/elasticsearch/common/cli/CliTool.java index bea819151f9..0d32b6e2779 100644 --- a/core/src/main/java/org/elasticsearch/common/cli/CliTool.java +++ b/core/src/main/java/org/elasticsearch/common/cli/CliTool.java @@ -22,7 +22,6 @@ package org.elasticsearch.common.cli; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; -import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.node.internal.InternalSettingsPreparer; @@ -100,7 +99,9 @@ public abstract class CliTool { } protected CliTool(CliToolConfig config, Terminal terminal) { - Preconditions.checkArgument(config.cmds().size() != 0, "At least one command must be configured"); + if (config.cmds().size() == 0) { + throw new IllegalArgumentException("At least one command must be configured"); + } this.config = config; this.terminal = terminal; env = InternalSettingsPreparer.prepareEnvironment(EMPTY_SETTINGS, terminal); diff --git a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java index fd288e859e7..eab20aaa23b 100644 --- a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java +++ b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.collect; import com.google.common.collect.UnmodifiableIterator; import org.apache.lucene.util.mutable.MutableValueInt; -import org.elasticsearch.common.Preconditions; import java.lang.reflect.Array; import java.util.*; @@ -474,7 +473,9 @@ public final class CopyOnWriteHashMap extends AbstractMap { @Override public V get(Object key) { - Preconditions.checkArgument(key != null, "Null keys are not supported"); + if (key == null) { + throw new IllegalArgumentException("null keys are not supported"); + } final int hash = key.hashCode(); return root.get(key, hash); } @@ -490,8 +491,12 @@ public final class CopyOnWriteHashMap extends AbstractMap { * of the hash table. The current hash table is not modified. */ public CopyOnWriteHashMap copyAndPut(K key, V value) { - Preconditions.checkArgument(key != null, "null keys are not supported"); - Preconditions.checkArgument(value != null, "null values are not supported"); + if (key == null) { + throw new IllegalArgumentException("null keys are not supported"); + } + if (value == null) { + throw new IllegalArgumentException("null values are not supported"); + } final int hash = key.hashCode(); final MutableValueInt newValue = new MutableValueInt(); final InnerNode newRoot = root.put(key, hash, TOTAL_HASH_BITS, value, newValue); @@ -522,7 +527,9 @@ public final class CopyOnWriteHashMap extends AbstractMap { * Remove the given key from this map. The current hash table is not modified. */ public CopyOnWriteHashMap copyAndRemove(Object key) { - Preconditions.checkArgument(key != null, "Null keys are not supported"); + if (key == null) { + throw new IllegalArgumentException("null keys are not supported"); + } final int hash = key.hashCode(); final InnerNode newRoot = root.remove(key, hash); if (root == newRoot) { diff --git a/core/src/main/java/org/elasticsearch/common/inject/CreationException.java b/core/src/main/java/org/elasticsearch/common/inject/CreationException.java index 05c5f04823e..5c44111e600 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/CreationException.java +++ b/core/src/main/java/org/elasticsearch/common/inject/CreationException.java @@ -22,8 +22,6 @@ import org.elasticsearch.common.inject.spi.Message; import java.util.Collection; -import static org.elasticsearch.common.Preconditions.checkArgument; - /** * Thrown when errors occur while creating a {@link Injector}. Includes a list of encountered * errors. Clients should catch this exception, log it, and stop execution. @@ -39,7 +37,9 @@ public class CreationException extends RuntimeException { */ public CreationException(Collection messages) { this.messages = ImmutableSet.copyOf(messages); - checkArgument(!this.messages.isEmpty()); + if (this.messages.isEmpty()) { + throw new IllegalArgumentException(); + } initCause(Errors.getOnlyCause(this.messages)); } diff --git a/core/src/main/java/org/elasticsearch/common/inject/Key.java b/core/src/main/java/org/elasticsearch/common/inject/Key.java index 8ee801ccea3..92925c69b43 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/Key.java +++ b/core/src/main/java/org/elasticsearch/common/inject/Key.java @@ -24,8 +24,6 @@ import java.lang.annotation.Annotation; import java.lang.reflect.Type; import java.util.Objects; -import static org.elasticsearch.common.Preconditions.checkArgument; - /** * Binding key consisting of an injection type and an optional annotation. * Matches the type and annotation at a point of injection. @@ -367,16 +365,20 @@ public class Key { private static void ensureRetainedAtRuntime( Class annotationType) { - checkArgument(Annotations.isRetainedAtRuntime(annotationType), - "%s is not retained at runtime. Please annotate it with @Retention(RUNTIME).", - annotationType.getName()); + if (!Annotations.isRetainedAtRuntime(annotationType)) { + throw new IllegalArgumentException( + annotationType.getName() + " is not retained at runtime. Please annotate it with @Retention(RUNTIME)." + ); + } } private static void ensureIsBindingAnnotation( Class annotationType) { - checkArgument(isBindingAnnotation(annotationType), - "%s is not a binding annotation. Please annotate it with @BindingAnnotation.", - annotationType.getName()); + if (!isBindingAnnotation(annotationType)) { + throw new IllegalArgumentException( + annotationType.getName() + " is not a binding annotation. Please annotate it with @BindingAnnotation." + ); + } } static enum NullAnnotationStrategy implements AnnotationStrategy { diff --git a/core/src/main/java/org/elasticsearch/common/inject/ProvisionException.java b/core/src/main/java/org/elasticsearch/common/inject/ProvisionException.java index dea1bfba5ad..9497169d179 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/ProvisionException.java +++ b/core/src/main/java/org/elasticsearch/common/inject/ProvisionException.java @@ -23,8 +23,6 @@ import org.elasticsearch.common.inject.spi.Message; import java.util.Collection; import java.util.Collections; -import static org.elasticsearch.common.Preconditions.checkArgument; - /** * Indicates that there was a runtime failure while providing an instance. * @@ -41,7 +39,9 @@ public final class ProvisionException extends RuntimeException { */ public ProvisionException(Iterable messages) { this.messages = ImmutableSet.copyOf(messages); - checkArgument(!this.messages.isEmpty()); + if (this.messages.isEmpty()) { + throw new IllegalArgumentException(); + } initCause(Errors.getOnlyCause(this.messages)); } diff --git a/core/src/main/java/org/elasticsearch/common/inject/TypeLiteral.java b/core/src/main/java/org/elasticsearch/common/inject/TypeLiteral.java index 44c89e4a3c4..ee50b129dc3 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/TypeLiteral.java +++ b/core/src/main/java/org/elasticsearch/common/inject/TypeLiteral.java @@ -24,7 +24,6 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; -import static org.elasticsearch.common.Preconditions.checkArgument; import static org.elasticsearch.common.inject.internal.MoreTypes.canonicalize; /** @@ -257,8 +256,9 @@ public class TypeLiteral { * @since 2.0 */ public TypeLiteral getSupertype(Class supertype) { - checkArgument(supertype.isAssignableFrom(rawType), - "%s is not a supertype of %s", supertype, this.type); + if (!supertype.isAssignableFrom(rawType)) { + throw new IllegalArgumentException(supertype + " is not a supertype of " + type); + } return resolve(MoreTypes.getGenericSupertype(type, rawType, supertype)); } @@ -269,8 +269,9 @@ public class TypeLiteral { * @since 2.0 */ public TypeLiteral getFieldType(Field field) { - checkArgument(field.getDeclaringClass().isAssignableFrom(rawType), - "%s is not defined by a supertype of %s", field, type); + if (!field.getDeclaringClass().isAssignableFrom(rawType)) { + throw new IllegalArgumentException(field + " is not defined by a supertype of " + type); + } return resolve(field.getGenericType()); } @@ -285,14 +286,17 @@ public class TypeLiteral { if (methodOrConstructor instanceof Method) { Method method = (Method) methodOrConstructor; - checkArgument(method.getDeclaringClass().isAssignableFrom(rawType), - "%s is not defined by a supertype of %s", method, type); + if (!method.getDeclaringClass().isAssignableFrom(rawType)) { + throw new IllegalArgumentException(method + " is not defined by a supertype of " + type); + } genericParameterTypes = method.getGenericParameterTypes(); } else if (methodOrConstructor instanceof Constructor) { Constructor constructor = (Constructor) methodOrConstructor; - checkArgument(constructor.getDeclaringClass().isAssignableFrom(rawType), - "%s does not construct a supertype of %s", constructor, type); + if (!constructor.getDeclaringClass().isAssignableFrom(rawType)) { + throw new IllegalArgumentException(constructor + " does not construct a supertype of " + type); + } + genericParameterTypes = constructor.getGenericParameterTypes(); } else { @@ -313,14 +317,17 @@ public class TypeLiteral { if (methodOrConstructor instanceof Method) { Method method = (Method) methodOrConstructor; - checkArgument(method.getDeclaringClass().isAssignableFrom(rawType), - "%s is not defined by a supertype of %s", method, type); + if (!method.getDeclaringClass().isAssignableFrom(rawType)) { + throw new IllegalArgumentException(method + " is not defined by a supertype of " + type); + } + genericExceptionTypes = method.getGenericExceptionTypes(); } else if (methodOrConstructor instanceof Constructor) { Constructor constructor = (Constructor) methodOrConstructor; - checkArgument(constructor.getDeclaringClass().isAssignableFrom(rawType), - "%s does not construct a supertype of %s", constructor, type); + if (!constructor.getDeclaringClass().isAssignableFrom(rawType)) { + throw new IllegalArgumentException(constructor + " does not construct a supertype of " + type); + } genericExceptionTypes = constructor.getGenericExceptionTypes(); } else { @@ -337,8 +344,10 @@ public class TypeLiteral { * @since 2.0 */ public TypeLiteral getReturnType(Method method) { - checkArgument(method.getDeclaringClass().isAssignableFrom(rawType), - "%s is not defined by a supertype of %s", method, type); + if (!method.getDeclaringClass().isAssignableFrom(rawType)) { + throw new IllegalArgumentException(method + " is not defined by a supertype of " + type); + } + return resolve(method.getGenericReturnType()); } } diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java index 1872a419a76..16e4deb65df 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java +++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java @@ -22,8 +22,6 @@ import java.lang.annotation.Annotation; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; -import static org.elasticsearch.common.Preconditions.checkArgument; - /** * Models a method or constructor parameter. * @@ -144,8 +142,9 @@ class Parameter { Annotation bindingAnnotation = null; for (Annotation a : annotations) { if (a.annotationType().getAnnotation(BindingAnnotation.class) != null) { - checkArgument(bindingAnnotation == null, - "Parameter has multiple binding annotations: %s and %s", bindingAnnotation, a); + if (bindingAnnotation != null) { + throw new IllegalArgumentException("Parameter has multiple binding annotations: " + bindingAnnotation + " and " + a); + } bindingAnnotation = a; } } diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/MoreTypes.java b/core/src/main/java/org/elasticsearch/common/inject/internal/MoreTypes.java index f156a389501..2a6111674e7 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/internal/MoreTypes.java +++ b/core/src/main/java/org/elasticsearch/common/inject/internal/MoreTypes.java @@ -30,8 +30,6 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; -import static org.elasticsearch.common.Preconditions.checkArgument; - /** * Static methods for working with types that we aren't publishing in the * public {@code Types} API. @@ -152,8 +150,11 @@ public class MoreTypes { // Neal isn't either but suspects some pathological case related // to nested classes exists. Type rawType = parameterizedType.getRawType(); - checkArgument(rawType instanceof Class, - "Expected a Class, but <%s> is of type %s", type, type.getClass().getName()); + if (!(rawType instanceof Class)) { + throw new IllegalArgumentException( + "Expected a Class, but <" + type +"> is of type " + type.getClass().getName() + ); + } return (Class) rawType; } else if (type instanceof GenericArrayType) { @@ -445,10 +446,13 @@ public class MoreTypes { // require an owner type if the raw type needs it if (rawType instanceof Class) { Class rawTypeAsClass = (Class) rawType; - checkArgument(ownerType != null || rawTypeAsClass.getEnclosingClass() == null, - "No owner type for enclosed %s", rawType); - checkArgument(ownerType == null || rawTypeAsClass.getEnclosingClass() != null, - "Owner type for unenclosed %s", rawType); + if (ownerType == null && rawTypeAsClass.getEnclosingClass() != null) { + throw new IllegalArgumentException("No owner type for enclosed " + rawType); + } + if (ownerType != null && rawTypeAsClass.getEnclosingClass() == null) { + throw new IllegalArgumentException("Owner type for unenclosed " + rawType); + } + } this.ownerType = ownerType == null ? null : canonicalize(ownerType); @@ -561,13 +565,18 @@ public class MoreTypes { private final Type lowerBound; public WildcardTypeImpl(Type[] upperBounds, Type[] lowerBounds) { - checkArgument(lowerBounds.length <= 1, "Must have at most one lower bound."); - checkArgument(upperBounds.length == 1, "Must have exactly one upper bound."); - + if (lowerBounds.length > 1) { + throw new IllegalArgumentException("Must have at most one lower bound."); + } + if (upperBounds.length != 1) { + throw new IllegalArgumentException("Must have exactly one upper bound."); + } if (lowerBounds.length == 1) { Objects.requireNonNull(lowerBounds[0], "lowerBound"); checkNotPrimitive(lowerBounds[0], "wildcard bounds"); - checkArgument(upperBounds[0] == Object.class, "bounded both ways"); + if (upperBounds[0] != Object.class) { + throw new IllegalArgumentException("bounded both ways"); + } this.lowerBound = canonicalize(lowerBounds[0]); this.upperBound = Object.class; @@ -615,8 +624,9 @@ public class MoreTypes { } private static void checkNotPrimitive(Type type, String use) { - checkArgument(!(type instanceof Class) || !((Class) type).isPrimitive(), - "Primitive types are not allowed in %s: %s", use, type); + if (type instanceof Class && ((Class) type).isPrimitive()) { + throw new IllegalArgumentException("Primitive types are not allowed in " + use + ": " + type); + } } /** diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java b/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java index 125c4d367af..6ab5454325a 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java +++ b/core/src/main/java/org/elasticsearch/common/inject/internal/PrivateElementsImpl.java @@ -27,8 +27,6 @@ import org.elasticsearch.common.inject.spi.PrivateElements; import java.util.*; -import static org.elasticsearch.common.Preconditions.checkArgument; - /** * @author jessewilson@google.com (Jesse Wilson) */ @@ -132,7 +130,9 @@ public final class PrivateElementsImpl implements PrivateElements { public Object getExposedSource(Key key) { getExposedKeys(); // ensure exposedKeysToSources is populated Object source = exposedKeysToSources.get(key); - checkArgument(source != null, "%s not exposed by %s.", key, this); + if (source == null) { + throw new IllegalArgumentException(key + " not exposed by " + "."); + } return source; } diff --git a/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java b/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java index b502b28c61c..c29a48ddd91 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java +++ b/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java @@ -24,8 +24,6 @@ import java.lang.reflect.AnnotatedElement; import java.lang.reflect.Method; import java.util.Objects; -import static org.elasticsearch.common.Preconditions.checkArgument; - /** * Matcher implementations. Supports matching classes and methods. * @@ -103,8 +101,9 @@ public class Matchers { private static void checkForRuntimeRetention( Class annotationType) { Retention retention = annotationType.getAnnotation(Retention.class); - checkArgument(retention != null && retention.value() == RetentionPolicy.RUNTIME, - "Annotation " + annotationType.getSimpleName() + " is missing RUNTIME retention"); + if (retention == null || retention.value() != RetentionPolicy.RUNTIME) { + throw new IllegalArgumentException("Annotation " + annotationType.getSimpleName() + " is missing RUNTIME retention"); + } } /** diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java b/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java index c8786d375cc..afc85458df0 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java +++ b/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java @@ -28,8 +28,6 @@ import org.elasticsearch.common.logging.Loggers; import java.lang.annotation.Annotation; import java.util.*; -import static org.elasticsearch.common.Preconditions.checkArgument; - /** * Exposes elements of a module so they can be inspected, validated or {@link * Element#applyTo(Binder) rewritten}. @@ -132,7 +130,9 @@ public final class Elements { */ private RecordingBinder( RecordingBinder prototype, Object source, SourceProvider sourceProvider) { - checkArgument(source == null ^ sourceProvider == null); + if (!(source == null ^ sourceProvider == null)) { + throw new IllegalArgumentException(); + } this.stage = prototype.stage; this.modules = prototype.modules; diff --git a/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java b/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java index 60335ed49fb..3b4a0733ec5 100644 --- a/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java +++ b/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java @@ -19,7 +19,6 @@ package org.elasticsearch.common.unit; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; @@ -44,7 +43,9 @@ public final class Fuzziness implements ToXContent { private final String fuzziness; private Fuzziness(int fuzziness) { - Preconditions.checkArgument(fuzziness >= 0 && fuzziness <= 2, "Valid edit distances are [0, 1, 2] but was [" + fuzziness + "]"); + if (fuzziness != 0 && fuzziness != 1 && fuzziness != 2) { + throw new IllegalArgumentException("Valid edit distances are [0, 1, 2] but was [" + fuzziness + "]"); + } this.fuzziness = Integer.toString(fuzziness); } diff --git a/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java b/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java index bc7d913d9d8..fcbcff3c3d7 100644 --- a/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java +++ b/core/src/main/java/org/elasticsearch/common/unit/SizeValue.java @@ -20,7 +20,6 @@ package org.elasticsearch.common.unit; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -46,7 +45,9 @@ public class SizeValue implements Streamable { } public SizeValue(long size, SizeUnit sizeUnit) { - Preconditions.checkArgument(size >= 0, "size in SizeValue may not be negative"); + if (size < 0) { + throw new IllegalArgumentException("size in SizeValue may not be negative"); + } this.size = size; this.sizeUnit = sizeUnit; } diff --git a/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java b/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java index 2328c8bc65e..29beee99d5a 100644 --- a/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java @@ -22,7 +22,6 @@ package org.elasticsearch.common.util; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.cache.recycler.PageCacheRecycler; -import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.recycler.Recycler; @@ -42,8 +41,12 @@ abstract class AbstractBigArray extends AbstractArray { protected AbstractBigArray(int pageSize, BigArrays bigArrays, boolean clearOnResize) { super(bigArrays, clearOnResize); this.recycler = bigArrays.recycler; - Preconditions.checkArgument(pageSize >= 128, "pageSize must be >= 128"); - Preconditions.checkArgument((pageSize & (pageSize - 1)) == 0, "pageSize must be a power of two"); + if (pageSize < 128) { + throw new IllegalArgumentException("pageSize must be >= 128"); + } + if ((pageSize & (pageSize - 1)) != 0) { + throw new IllegalArgumentException("pageSize must be a power of two"); + } this.pageShift = Integer.numberOfTrailingZeros(pageSize); this.pageMask = pageSize - 1; size = 0; @@ -56,7 +59,9 @@ abstract class AbstractBigArray extends AbstractArray { final int numPages(long capacity) { final long numPages = (capacity + pageMask) >>> pageShift; - Preconditions.checkArgument(numPages <= Integer.MAX_VALUE, "pageSize=" + (pageMask + 1) + " is too small for such as capacity: " + capacity); + if (numPages > Integer.MAX_VALUE) { + throw new IllegalArgumentException("pageSize=" + (pageMask + 1) + " is too small for such as capacity: " + capacity); + } return (int) numPages; } diff --git a/core/src/main/java/org/elasticsearch/common/util/AbstractPagedHashMap.java b/core/src/main/java/org/elasticsearch/common/util/AbstractPagedHashMap.java index 7b1298beab8..2368a5dde0a 100644 --- a/core/src/main/java/org/elasticsearch/common/util/AbstractPagedHashMap.java +++ b/core/src/main/java/org/elasticsearch/common/util/AbstractPagedHashMap.java @@ -20,7 +20,6 @@ package org.elasticsearch.common.util; import com.carrotsearch.hppc.BitMixer; -import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.lease.Releasable; /** @@ -48,8 +47,12 @@ abstract class AbstractPagedHashMap implements Releasable { long mask; AbstractPagedHashMap(long capacity, float maxLoadFactor, BigArrays bigArrays) { - Preconditions.checkArgument(capacity >= 0, "capacity must be >= 0"); - Preconditions.checkArgument(maxLoadFactor > 0 && maxLoadFactor < 1, "maxLoadFactor must be > 0 and < 1"); + if (capacity < 0) { + throw new IllegalArgumentException("capacity must be >= 0"); + } + if (maxLoadFactor <= 0 || maxLoadFactor >= 1) { + throw new IllegalArgumentException("maxLoadFactor must be > 0 and < 1"); + } this.bigArrays = bigArrays; this.maxLoadFactor = maxLoadFactor; long buckets = 1L + (long) (capacity / maxLoadFactor); diff --git a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java index 49e338ef87b..0f0bced9cf2 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java @@ -24,7 +24,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.cache.recycler.PageCacheRecycler; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.inject.Inject; @@ -55,9 +54,15 @@ public class BigArrays { /** Return the next size to grow to that is >= minTargetSize. * Inspired from {@link ArrayUtil#oversize(int, int)} and adapted to play nicely with paging. */ public static long overSize(long minTargetSize, int pageSize, int bytesPerElement) { - Preconditions.checkArgument(minTargetSize >= 0, "minTargetSize must be >= 0"); - Preconditions.checkArgument(pageSize >= 0, "pageSize must be > 0"); - Preconditions.checkArgument(bytesPerElement > 0, "bytesPerElement must be > 0"); + if (minTargetSize < 0) { + throw new IllegalArgumentException("minTargetSize must be >= 0"); + } + if (pageSize < 0) { + throw new IllegalArgumentException("pageSize must be > 0"); + } + if (bytesPerElement <= 0) { + throw new IllegalArgumentException("bytesPerElement must be > 0"); + } long newSize; if (minTargetSize < pageSize) { diff --git a/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java b/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java index 7b7c69de9e3..da4bc28408d 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java @@ -22,7 +22,6 @@ package org.elasticsearch.common.util; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; -import org.elasticsearch.common.Preconditions; import java.util.Arrays; @@ -110,7 +109,9 @@ final class BigByteArray extends AbstractBigArray implements ByteArray { @Override public void fill(long fromIndex, long toIndex, byte value) { - Preconditions.checkArgument(fromIndex <= toIndex); + if (fromIndex > toIndex) { + throw new IllegalArgumentException(); + } final int fromPage = pageIndex(fromIndex); final int toPage = pageIndex(toIndex - 1); if (fromPage == toPage) { diff --git a/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java b/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java index e12138deafc..1f739188377 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.util; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; -import org.elasticsearch.common.Preconditions; import java.util.Arrays; @@ -94,7 +93,9 @@ final class BigDoubleArray extends AbstractBigArray implements DoubleArray { @Override public void fill(long fromIndex, long toIndex, double value) { - Preconditions.checkArgument(fromIndex <= toIndex); + if (fromIndex > toIndex) { + throw new IllegalArgumentException(); + } final long longBits = Double.doubleToRawLongBits(value); final int fromPage = pageIndex(fromIndex); final int toPage = pageIndex(toIndex - 1); diff --git a/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java b/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java index a56ac31a149..f6fc2d8fce0 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.util; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; -import org.elasticsearch.common.Preconditions; import java.util.Arrays; @@ -94,7 +93,9 @@ final class BigFloatArray extends AbstractBigArray implements FloatArray { @Override public void fill(long fromIndex, long toIndex, float value) { - Preconditions.checkArgument(fromIndex <= toIndex); + if (fromIndex > toIndex) { + throw new IllegalArgumentException(); + } final int intBits = Float.floatToRawIntBits(value); final int fromPage = pageIndex(fromIndex); final int toPage = pageIndex(toIndex - 1); diff --git a/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java b/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java index 82bc451df29..1c0e9fe017c 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.util; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; -import org.elasticsearch.common.Preconditions; import java.util.Arrays; @@ -71,7 +70,9 @@ final class BigIntArray extends AbstractBigArray implements IntArray { @Override public void fill(long fromIndex, long toIndex, int value) { - Preconditions.checkArgument(fromIndex <= toIndex); + if (fromIndex > toIndex) { + throw new IllegalArgumentException(); + } final int fromPage = pageIndex(fromIndex); final int toPage = pageIndex(toIndex - 1); if (fromPage == toPage) { diff --git a/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java b/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java index 57750b3b2c9..fe0323ba67c 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java +++ b/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java @@ -21,7 +21,6 @@ package org.elasticsearch.common.util; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; -import org.elasticsearch.common.Preconditions; import java.util.Arrays; @@ -93,7 +92,9 @@ final class BigLongArray extends AbstractBigArray implements LongArray { @Override public void fill(long fromIndex, long toIndex, long value) { - Preconditions.checkArgument(fromIndex <= toIndex); + if (fromIndex > toIndex) { + throw new IllegalArgumentException(); + } if (fromIndex == toIndex) { return; // empty range } diff --git a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java index 6f3d701a050..5d520ffbbd4 100644 --- a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java +++ b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java @@ -25,7 +25,6 @@ import com.carrotsearch.hppc.LongArrayList; import com.carrotsearch.hppc.ObjectArrayList; import com.google.common.collect.Iterators; import org.apache.lucene.util.*; -import org.elasticsearch.common.Preconditions; import java.util.*; @@ -284,8 +283,12 @@ public enum CollectionUtils { private final int distance; public RotatedList(List list, int distance) { - Preconditions.checkArgument(distance >= 0 && distance < list.size()); - Preconditions.checkArgument(list instanceof RandomAccess); + if (distance < 0 || distance >= list.size()) { + throw new IllegalArgumentException(); + } + if (!(list instanceof RandomAccess)) { + throw new IllegalArgumentException(); + } this.in = list; this.distance = distance; } diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index 86a849b82cc..9ac1768522c 100644 --- a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.*; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.store.IndexOutputOutputStream; @@ -90,8 +89,12 @@ public abstract class MetaDataStateFormat { * @throws IOException if an IOException occurs */ public final void write(final T state, final long version, final Path... locations) throws IOException { - Preconditions.checkArgument(locations != null, "Locations must not be null"); - Preconditions.checkArgument(locations.length > 0, "One or more locations required"); + if (locations == null) { + throw new IllegalArgumentException("Locations must not be null"); + } + if (locations.length <= 0) { + throw new IllegalArgumentException("One or more locations required"); + } final long maxStateId = findMaxStateId(prefix, locations)+1; assert maxStateId >= 0 : "maxStateId must be positive but was: [" + maxStateId + "]"; final String fileName = prefix + maxStateId + STATE_FILE_EXTENSION; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java index e00f7acde8d..86966617fcc 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java @@ -27,7 +27,6 @@ import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.fielddata.AtomicNumericFieldData; @@ -51,7 +50,9 @@ public class BinaryDVNumericIndexFieldData extends DocValuesIndexFieldData imple public BinaryDVNumericIndexFieldData(Index index, Names fieldNames, NumericType numericType, FieldDataType fieldDataType) { super(index, fieldNames, fieldDataType); - Preconditions.checkArgument(numericType != null, "numericType must be non-null"); + if (numericType == null) { + throw new IllegalArgumentException("numericType must be non-null"); + } this.numericType = numericType; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java index 42cba5bf6d8..1b54e386c0c 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java @@ -26,7 +26,6 @@ import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; @@ -73,7 +72,9 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData visitedOrds; OrdinalsCollector(HyperLogLogPlusPlus counts, RandomAccessOrds values, BigArrays bigArrays) { - Preconditions.checkArgument(values.getValueCount() <= Integer.MAX_VALUE); + if (values.getValueCount() > Integer.MAX_VALUE) { + throw new IllegalArgumentException(); + } maxOrd = (int) values.getValueCount(); this.bigArrays = bigArrays; this.counts = counts; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java index 3293f2372e7..6b389182fc7 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java @@ -23,7 +23,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongBitSet; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.packed.PackedInts; -import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lease.Releasable; @@ -160,8 +159,12 @@ public final class HyperLogLogPlusPlus implements Releasable { private final double alphaMM; public HyperLogLogPlusPlus(int precision, BigArrays bigArrays, long initialBucketCount) { - Preconditions.checkArgument(precision >= 4, "precision must be >= 4"); - Preconditions.checkArgument(precision <= 18, "precision must be <= 18"); + if (precision < 4) { + throw new IllegalArgumentException("precision must be >= 4"); + } + if (precision > 18) { + throw new IllegalArgumentException("precision must be <= 18"); + } p = precision; m = 1 << p; this.bigArrays = bigArrays; @@ -196,7 +199,9 @@ public final class HyperLogLogPlusPlus implements Releasable { } public void merge(long thisBucket, HyperLogLogPlusPlus other, long otherBucket) { - Preconditions.checkArgument(p == other.p); + if (p != other.p) { + throw new IllegalArgumentException(); + } ensureCapacity(thisBucket + 1); if (other.algorithm.get(otherBucket) == LINEAR_COUNTING) { final IntArray values = other.hashSet.values(otherBucket); From 527ab95c39d4960b48e33244af1bb25f70a7145b Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 14 Sep 2015 11:58:48 -0400 Subject: [PATCH 12/33] Remove and forbid use of com.google.common.collect.Iterables This commit removes and now forbids all uses of com.google.common.collect.Iterables across the codebase. This is one of many steps in the eventual removal of Guava as a dependency. Relates #13224 --- .../cluster/routing/RoutingNodes.java | 13 +- .../cluster/routing/RoutingTable.java | 13 +- .../service/InternalClusterService.java | 7 +- .../org/elasticsearch/common/Strings.java | 18 +-- .../common/inject/InjectorBuilder.java | 2 +- .../assistedinject/FactoryProvider2.java | 3 +- .../inject/internal/SourceProvider.java | 2 +- .../elasticsearch/common/lucene/Lucene.java | 9 +- .../common/util/CollectionUtils.java | 1 + .../common/util/iterable/Iterables.java | 142 ++++++++++++++++++ .../index/mapper/internal/IdFieldMapper.java | 18 +-- .../index/mapper/object/ObjectMapper.java | 24 +-- .../index/query/IdsQueryParser.java | 3 +- .../BlobStoreIndexShardRepository.java | 14 +- .../org/elasticsearch/index/store/Store.java | 2 +- .../flush/IndicesSyncedFlushResult.java | 4 +- .../recovery/RecoverySourceHandler.java | 6 +- .../search/aggregations/BucketCollector.java | 6 +- .../aggregations/LeafBucketCollector.java | 9 +- .../search/internal/SearchContext.java | 4 +- .../context/CategoryContextMapping.java | 10 +- .../suggest/context/ContextMapping.java | 11 +- .../common/util/CollectionUtilsTests.java | 12 +- .../common/util/iterable/IterablesTests.java | 81 ++++++++++ .../search/highlight/HighlighterSearchIT.java | 45 +----- .../resources/forbidden/all-signatures.txt | 3 +- 26 files changed, 286 insertions(+), 176 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java create mode 100644 core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index 0a2a5c96606..9c9cfaa27a1 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -21,7 +21,6 @@ package org.elasticsearch.cluster.routing; import com.carrotsearch.hppc.ObjectIntHashMap; import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.cluster.ClusterState; @@ -31,15 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.index.shard.ShardId; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.function.Predicate; /** @@ -308,7 +299,7 @@ public class RoutingNodes implements Iterable { } for (ShardRoutingState s : state) { if (s == ShardRoutingState.UNASSIGNED) { - Iterables.addAll(shards, unassigned()); + unassigned().forEach(shards::add); break; } } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index 8f6b05b9c77..07fe33acad9 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -21,7 +21,6 @@ package org.elasticsearch.cluster.routing; import com.carrotsearch.hppc.IntSet; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.common.collect.UnmodifiableIterator; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.Diffable; @@ -30,14 +29,11 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.index.IndexNotFoundException; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Predicate; /** @@ -379,7 +375,10 @@ public class RoutingTable implements Iterable, Diffable shardRoutingEntries = Iterables.concat(routingNodes.unassigned(), routingNodes.unassigned().ignored()); + + for (ShardRouting shardRoutingEntry : shardRoutingEntries) { String index = shardRoutingEntry.index(); IndexRoutingTable.Builder indexBuilder = indexRoutingTableBuilders.get(index); if (indexBuilder == null) { diff --git a/core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java index 08cdbbb863b..2b8a180dece 100644 --- a/core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java +++ b/core/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java @@ -19,7 +19,6 @@ package org.elasticsearch.cluster.service; -import com.google.common.collect.Iterables; import org.elasticsearch.Version; import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.ClusterState.Builder; @@ -42,6 +41,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.StringText; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.common.util.concurrent.*; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.DiscoveryService; @@ -89,10 +89,7 @@ public class InternalClusterService extends AbstractLifecycleComponent lastClusterStateListeners = new CopyOnWriteArrayList<>(); // TODO this is rather frequently changing I guess a Synced Set would be better here and a dedicated remove API private final Collection postAppliedListeners = new CopyOnWriteArrayList<>(); - private final Iterable preAppliedListeners = Iterables.concat( - priorityClusterStateListeners, - clusterStateListeners, - lastClusterStateListeners); + private final Iterable preAppliedListeners = Iterables.concat(priorityClusterStateListeners, clusterStateListeners, lastClusterStateListeners); private final LocalNodeMasterListeners localNodeMasterListeners; diff --git a/core/src/main/java/org/elasticsearch/common/Strings.java b/core/src/main/java/org/elasticsearch/common/Strings.java index dafaa616d57..40d9602e2cd 100644 --- a/core/src/main/java/org/elasticsearch/common/Strings.java +++ b/core/src/main/java/org/elasticsearch/common/Strings.java @@ -20,8 +20,6 @@ package org.elasticsearch.common; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; - import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.FastStringReader; @@ -32,18 +30,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import java.io.BufferedReader; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; -import java.util.Random; -import java.util.Set; -import java.util.StringTokenizer; -import java.util.TreeSet; +import java.util.*; /** * @@ -840,9 +827,6 @@ public class Strings { } public static String collectionToDelimitedString(Iterable coll, String delim, String prefix, String suffix, StringBuilder sb) { - if (Iterables.isEmpty(coll)) { - return ""; - } Iterator it = coll.iterator(); while (it.hasNext()) { sb.append(prefix).append(it.next()).append(suffix); diff --git a/core/src/main/java/org/elasticsearch/common/inject/InjectorBuilder.java b/core/src/main/java/org/elasticsearch/common/inject/InjectorBuilder.java index 32d7477c89e..5f7175dcfec 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/InjectorBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/inject/InjectorBuilder.java @@ -17,9 +17,9 @@ package org.elasticsearch.common.inject; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; import org.elasticsearch.common.inject.internal.*; import org.elasticsearch.common.inject.spi.Dependency; +import org.elasticsearch.common.util.iterable.Iterables; import java.util.Collection; import java.util.List; diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java index b1d6b400831..bedd7969cc8 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java +++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java @@ -17,7 +17,6 @@ package org.elasticsearch.common.inject.assistedinject; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import org.elasticsearch.common.inject.*; import org.elasticsearch.common.inject.internal.Errors; import org.elasticsearch.common.inject.internal.ErrorsException; @@ -227,7 +226,7 @@ public final class FactoryProvider2 implements InvocationHandler, Provider } catch (ProvisionException e) { // if this is an exception declared by the factory method, throw it as-is if (e.getErrorMessages().size() == 1) { - Message onlyError = Iterables.getOnlyElement(e.getErrorMessages()); + Message onlyError = e.getErrorMessages().iterator().next(); Throwable cause = onlyError.getCause(); if (cause != null && canRethrow(method, cause)) { throw cause; diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/SourceProvider.java b/core/src/main/java/org/elasticsearch/common/inject/internal/SourceProvider.java index f0fa05c66dd..a149c5739a0 100644 --- a/core/src/main/java/org/elasticsearch/common/inject/internal/SourceProvider.java +++ b/core/src/main/java/org/elasticsearch/common/inject/internal/SourceProvider.java @@ -17,7 +17,7 @@ package org.elasticsearch.common.inject.internal; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; +import org.elasticsearch.common.util.iterable.Iterables; import java.util.ArrayList; import java.util.List; diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 0d0db20a056..db2bb12cb8c 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -19,7 +19,6 @@ package org.elasticsearch.common.lucene; -import com.google.common.collect.Iterables; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.codecs.CodecUtil; @@ -27,7 +26,10 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.*; import org.apache.lucene.search.*; -import org.apache.lucene.store.*; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.Lock; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; @@ -40,6 +42,7 @@ import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -107,7 +110,7 @@ public class Lucene { for (SegmentCommitInfo info : infos) { list.add(info.files()); } - return Iterables.concat(list); + return Iterables.flatten(list); } /** diff --git a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java index 5d520ffbbd4..ec37fb21de1 100644 --- a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java +++ b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java @@ -452,4 +452,5 @@ public enum CollectionUtils { return result; } + } diff --git a/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java b/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java new file mode 100644 index 00000000000..f6b891c9997 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java @@ -0,0 +1,142 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.util.iterable; + +import org.elasticsearch.common.lucene.store.IndexOutputOutputStream; + +import java.util.*; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public class Iterables { + public Iterables() { + } + + public static Iterable concat(Iterable... inputs) { + Objects.requireNonNull(inputs); + return new ConcatenatedIterable(inputs); + } + + static class ConcatenatedIterable implements Iterable { + private final Iterable[] inputs; + + ConcatenatedIterable(Iterable[] inputs) { + this.inputs = Arrays.copyOf(inputs, inputs.length); + } + + @Override + public Iterator iterator() { + return Stream + .of(inputs) + .map(it -> StreamSupport.stream(it.spliterator(), false)) + .reduce(Stream::concat) + .orElseGet(Stream::empty).iterator(); + } + } + + public static Iterable flatten(Iterable> inputs) { + Objects.requireNonNull(inputs); + return new FlattenedIterables<>(inputs); + } + + static class FlattenedIterables implements Iterable { + private final Iterable> inputs; + + FlattenedIterables(Iterable> inputs) { + List> list = new ArrayList<>(); + for (Iterable iterable : inputs) { + list.add(iterable); + } + this.inputs = list; + } + + @Override + public Iterator iterator() { + return StreamSupport + .stream(inputs.spliterator(), false) + .flatMap(s -> StreamSupport.stream(s.spliterator(), false)).iterator(); + } + } + + public static boolean allElementsAreEqual(Iterable left, Iterable right) { + Objects.requireNonNull(left); + Objects.requireNonNull(right); + if (left instanceof Collection && right instanceof Collection) { + Collection collection1 = (Collection) left; + Collection collection2 = (Collection) right; + if (collection1.size() != collection2.size()) { + return false; + } + } + + Iterator leftIt = left.iterator(); + Iterator rightIt = right.iterator(); + + while (true) { + if (leftIt.hasNext()) { + if (!rightIt.hasNext()) { + return false; + } + + Object o1 = leftIt.next(); + Object o2 = rightIt.next(); + if (Objects.equals(o1, o2)) { + continue; + } + + return false; + } + + return !rightIt.hasNext(); + } + } + + public static T getFirst(Iterable collection, T defaultValue) { + Objects.requireNonNull(collection); + Iterator iterator = collection.iterator(); + return iterator.hasNext() ? iterator.next() : defaultValue; + } + + public static T get(Iterable iterable, int position) { + Objects.requireNonNull(iterable); + if (position < 0) { + throw new IllegalArgumentException("position >= 0"); + } + if (iterable instanceof List) { + List list = (List)iterable; + if (position >= list.size()) { + throw new IndexOutOfBoundsException(Integer.toString(position)); + } + return list.get(position); + } else { + Iterator it = iterable.iterator(); + for (int index = 0; index < position; index++) { + if (!it.hasNext()) { + throw new IndexOutOfBoundsException(Integer.toString(position)); + } + it.next(); + } + if (!it.hasNext()) { + throw new IndexOutOfBoundsException(Integer.toString(position)); + } + return it.next(); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java index 96810ec3b88..c21e07c4cfc 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java @@ -19,18 +19,12 @@ package org.elasticsearch.index.mapper.internal; -import com.google.common.collect.Iterables; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.queries.TermsQuery; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.MultiTermQuery; -import org.apache.lucene.search.PrefixQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.RegexpQuery; +import org.apache.lucene.search.*; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; @@ -38,17 +32,11 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.FieldDataType; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; -import org.elasticsearch.index.mapper.MergeResult; -import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.query.QueryParseContext; import java.io.IOException; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java index 3ff0611fca0..24be42c6094 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper.object; -import com.google.common.collect.Iterables; import org.apache.lucene.index.Term; import org.apache.lucene.search.Filter; import org.apache.lucene.search.QueryWrapperFilter; @@ -33,30 +32,13 @@ import org.elasticsearch.common.collect.CopyOnWriteHashMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.DocumentMapperParser; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MapperUtils; -import org.elasticsearch.index.mapper.MergeMappingException; -import org.elasticsearch.index.mapper.MergeResult; -import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.internal.AllFieldMapper; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.settings.IndexSettings; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; +import java.util.*; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; import static org.elasticsearch.index.mapper.MapperBuilders.object; @@ -583,7 +565,7 @@ public class ObjectMapper extends Mapper implements AllFieldMapper.IncludeInAll, doXContent(builder, params); // sort the mappers so we get consistent serialization format - Mapper[] sortedMappers = Iterables.toArray(mappers.values(), Mapper.class); + Mapper[] sortedMappers = mappers.values().stream().toArray(size -> new Mapper[size]); Arrays.sort(sortedMappers, new Comparator() { @Override public int compare(Mapper o1, Mapper o2) { diff --git a/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java index dcbb19f7bb7..62c2224dc68 100644 --- a/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java @@ -19,13 +19,12 @@ package org.elasticsearch.index.query; -import com.google.common.collect.Iterables; - import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.internal.UidFieldMapper; diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java index ed42e2f0ac0..947a36a6c1c 100644 --- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java +++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.snapshots.blobstore; -import com.google.common.collect.Iterables; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; @@ -48,14 +47,11 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.store.InputStreamIndexInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.deletionpolicy.SnapshotIndexCommit; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.snapshots.IndexShardRepository; -import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException; -import org.elasticsearch.index.snapshots.IndexShardSnapshotException; -import org.elasticsearch.index.snapshots.IndexShardSnapshotFailedException; -import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; +import org.elasticsearch.index.snapshots.*; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.store.StoreFileMetaData; @@ -71,11 +67,7 @@ import org.elasticsearch.repositories.blobstore.LegacyBlobStoreFormat; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.testBlobPrefix; diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java index 39a0f5365ba..147cf77c600 100644 --- a/core/src/main/java/org/elasticsearch/index/store/Store.java +++ b/core/src/main/java/org/elasticsearch/index/store/Store.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.store; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.*; import org.apache.lucene.store.*; @@ -42,6 +41,7 @@ import org.elasticsearch.common.lucene.store.InputStreamIndexInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.Callback; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.common.util.SingleObjectCache; import org.elasticsearch.common.util.concurrent.AbstractRefCounted; import org.elasticsearch.common.util.concurrent.RefCounted; diff --git a/core/src/main/java/org/elasticsearch/indices/flush/IndicesSyncedFlushResult.java b/core/src/main/java/org/elasticsearch/indices/flush/IndicesSyncedFlushResult.java index f625f04484a..54ec76e7a30 100644 --- a/core/src/main/java/org/elasticsearch/indices/flush/IndicesSyncedFlushResult.java +++ b/core/src/main/java/org/elasticsearch/indices/flush/IndicesSyncedFlushResult.java @@ -19,8 +19,8 @@ package org.elasticsearch.indices.flush; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; @@ -41,7 +41,7 @@ public class IndicesSyncedFlushResult implements ToXContent { public IndicesSyncedFlushResult(Map> shardsResultPerIndex) { this.shardsResultPerIndex = ImmutableMap.copyOf(shardsResultPerIndex); - this.shardCounts = calculateShardCounts(Iterables.concat(shardsResultPerIndex.values())); + this.shardCounts = calculateShardCounts(Iterables.flatten(shardsResultPerIndex.values())); } /** total number shards, including replicas, both assigned and unassigned */ diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index 8d913c6937f..b987b4a64a6 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -19,7 +19,6 @@ package org.elasticsearch.indices.recovery; -import com.google.common.collect.Iterables; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; @@ -39,6 +38,7 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.CancellableThreads; import org.elasticsearch.common.util.CancellableThreads.Interruptable; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.deletionpolicy.SnapshotIndexCommit; import org.elasticsearch.index.engine.Engine; @@ -64,6 +64,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.StreamSupport; /** * RecoverySourceHandler handles the three phases of shard recovery, which is @@ -411,7 +412,8 @@ public class RecoverySourceHandler { if ((corruptIndexException = ExceptionsHelper.unwrapCorruption(remoteException)) != null) { try { final Store.MetadataSnapshot recoverySourceMetadata = store.getMetadata(snapshot); - StoreFileMetaData[] metadata = Iterables.toArray(recoverySourceMetadata, StoreFileMetaData.class); + StoreFileMetaData[] metadata = + StreamSupport.stream(recoverySourceMetadata.spliterator(), false).toArray(size -> new StoreFileMetaData[size]); ArrayUtil.timSort(metadata, new Comparator() { @Override public int compare(StoreFileMetaData o1, StoreFileMetaData o2) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java index f862da6b3d6..ee38e2b3610 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java @@ -20,14 +20,13 @@ package org.elasticsearch.search.aggregations; -import com.google.common.collect.Iterables; - import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Collector; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.stream.StreamSupport; /** * A Collector that can collect data in separate buckets. @@ -58,7 +57,8 @@ public abstract class BucketCollector implements Collector { * Wrap the given collectors into a single instance. */ public static BucketCollector wrap(Iterable collectorList) { - final BucketCollector[] collectors = Iterables.toArray(collectorList, BucketCollector.class); + final BucketCollector[] collectors = + StreamSupport.stream(collectorList.spliterator(), false).toArray(size -> new BucketCollector[size]); switch (collectors.length) { case 0: return NO_OP_COLLECTOR; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/LeafBucketCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/LeafBucketCollector.java index eb51ba24af5..f5b7f15bb94 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/LeafBucketCollector.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/LeafBucketCollector.java @@ -19,12 +19,11 @@ package org.elasticsearch.search.aggregations; -import com.google.common.collect.Iterables; - import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Scorer; import java.io.IOException; +import java.util.stream.Stream; import java.util.stream.StreamSupport; /** @@ -44,9 +43,9 @@ public abstract class LeafBucketCollector implements LeafCollector { }; public static LeafBucketCollector wrap(Iterable collectors) { - final Iterable actualCollectors = - StreamSupport.stream(collectors.spliterator(), false).filter(c -> c != NO_OP_COLLECTOR)::iterator; - final LeafBucketCollector[] colls = Iterables.toArray(actualCollectors, LeafBucketCollector.class); + final Stream actualCollectors = + StreamSupport.stream(collectors.spliterator(), false).filter(c -> c != NO_OP_COLLECTOR); + final LeafBucketCollector[] colls = actualCollectors.toArray(size -> new LeafBucketCollector[size]); switch (colls.length) { case 0: return NO_OP_COLLECTOR; diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 234a841e017..34f1d5a8f45 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.search.internal; -import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; import com.google.common.collect.MultimapBuilder; @@ -35,6 +34,7 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; @@ -330,7 +330,7 @@ public abstract class SearchContext extends DelegatingHasContextAndHeaders imple } releasables.add(clearables.removeAll(lc)); } - Releasables.close(Iterables.concat(releasables)); + Releasables.close(Iterables.flatten(releasables)); } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java index ad075ac9b38..34b78301ca8 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.suggest.context; import com.google.common.base.Joiner; -import com.google.common.collect.Iterables; import org.apache.lucene.analysis.PrefixAnalyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.IndexableField; @@ -28,6 +27,7 @@ import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; @@ -35,11 +35,7 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParseContext.Document; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; /** * The {@link CategoryContextMapping} is used to define a {@link ContextMapping} that @@ -213,7 +209,7 @@ public class CategoryContextMapping extends ContextMapping { if (obj instanceof CategoryContextMapping) { CategoryContextMapping other = (CategoryContextMapping) obj; if (this.fieldName.equals(other.fieldName)) { - return Iterables.elementsEqual(this.defaultValues, other.defaultValues); + return Iterables.allElementsAreEqual(this.defaultValues, other.defaultValues); } } return false; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/context/ContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/context/ContextMapping.java index 849a6e5e7a5..65232cad395 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/context/ContextMapping.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/context/ContextMapping.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.suggest.context; -import com.google.common.collect.Iterables; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester; import org.apache.lucene.util.automaton.Automata; @@ -36,13 +35,7 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParseContext.Document; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; +import java.util.*; /** * A {@link ContextMapping} is used t define a context that may used @@ -157,7 +150,7 @@ public abstract class ContextMapping implements ToXContent { * @return true if both arguments are equal */ public static boolean mappingsAreEqual(SortedMap thisMappings, SortedMap otherMappings) { - return Iterables.elementsEqual(thisMappings.entrySet(), otherMappings.entrySet()); + return thisMappings.entrySet().equals(otherMappings.entrySet()); } @Override diff --git a/core/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java b/core/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java index b85c42c902a..73611c80be6 100644 --- a/core/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.common.util; -import com.google.common.collect.Iterables; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefArray; import org.apache.lucene.util.BytesRefBuilder; @@ -27,14 +26,7 @@ import org.apache.lucene.util.Counter; import org.elasticsearch.test.ESTestCase; import org.junit.Test; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; +import java.util.*; import static org.elasticsearch.common.util.CollectionUtils.eagerPartition; import static org.hamcrest.Matchers.equalTo; @@ -60,7 +52,7 @@ public class CollectionUtilsTests extends ESTestCase { final List rotated = CollectionUtils.rotate(list, distance); // check content is the same assertEquals(rotated.size(), list.size()); - assertEquals(Iterables.size(rotated), list.size()); + assertEquals(rotated.size(), list.size()); assertEquals(new HashSet<>(rotated), new HashSet<>(list)); // check stability for (int j = randomInt(4); j >= 0; --j) { diff --git a/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java b/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java new file mode 100644 index 00000000000..76fc51d8b67 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.util.iterable; + +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.object.HasToString.hasToString; +import static org.junit.Assert.*; + +public class IterablesTests extends ESTestCase { + public void testGetOverList() { + test(Arrays.asList("a", "b", "c")); + } + + public void testGetOverIterable() { + Iterable iterable = () -> + new Iterator() { + private int position = 0; + + @Override + public boolean hasNext() { + return position < 3; + } + + @Override + public String next() { + if (position < 3) { + String s = position == 0 ? "a" : position == 1 ? "b" : "c"; + position++; + return s; + } else { + throw new NoSuchElementException(); + } + } + }; + test(iterable); + } + + private void test(Iterable iterable) { + try { + Iterables.get(iterable, -1); + fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e, hasToString("java.lang.IllegalArgumentException: position >= 0")); + } + assertEquals("a", Iterables.get(iterable, 0)); + assertEquals("b", Iterables.get(iterable, 1)); + assertEquals("c", Iterables.get(iterable, 2)); + try { + Iterables.get(iterable, 3); + fail("expected IndexOutOfBoundsException"); + } catch (IndexOutOfBoundsException e) { + assertThat(e, hasToString("java.lang.IndexOutOfBoundsException: 3")); + } + } +} \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java index 999313736ad..93449c90963 100644 --- a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java @@ -19,22 +19,15 @@ package org.elasticsearch.search.highlight; import com.carrotsearch.randomizedtesting.generators.RandomPicks; -import com.google.common.base.Joiner; -import com.google.common.collect.Iterables; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings.Builder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.BoostableQueryBuilder; -import org.elasticsearch.index.query.IdsQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.MatchQueryBuilder.Operator; import org.elasticsearch.index.query.MatchQueryBuilder.Type; -import org.elasticsearch.index.query.MultiMatchQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -51,38 +44,12 @@ import java.util.Map; import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.boostingQuery; -import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery; -import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; -import static org.elasticsearch.index.query.QueryBuilders.fuzzyQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchPhrasePrefixQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.elasticsearch.index.query.QueryBuilders.missingQuery; -import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; -import static org.elasticsearch.index.query.QueryBuilders.prefixQuery; -import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; -import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; -import static org.elasticsearch.index.query.QueryBuilders.regexpQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.index.query.QueryBuilders.typeQuery; -import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; +import static org.elasticsearch.index.query.QueryBuilders.*; import static org.elasticsearch.search.builder.SearchSourceBuilder.highlight; import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHighlight; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNotHighlighted; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; import static org.elasticsearch.test.hamcrest.RegexMatcher.matches; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.startsWith; +import static org.hamcrest.Matchers.*; public class HighlighterSearchIT extends ESIntegTestCase { @@ -845,8 +812,10 @@ public class HighlighterSearchIT extends ESIntegTestCase { ensureGreen(); // Index one megabyte of "t " over and over and over again + String pattern = "t "; + String value = new String(new char[1024 * 256 / pattern.length()]).replace("\0", pattern); client().prepareIndex("test", "type1") - .setSource("field1", Joiner.on("").join(Iterables.limit(Iterables.cycle("t "), 1024*256))).get(); + .setSource("field1", value).get(); refresh(); logger.info("--> highlighting and searching on field1"); diff --git a/dev-tools/src/main/resources/forbidden/all-signatures.txt b/dev-tools/src/main/resources/forbidden/all-signatures.txt index d37cf2a35a5..82237cce048 100644 --- a/dev-tools/src/main/resources/forbidden/all-signatures.txt +++ b/dev-tools/src/main/resources/forbidden/all-signatures.txt @@ -110,7 +110,8 @@ com.google.common.base.Function com.google.common.collect.Collections2 com.google.common.cache.LoadingCache com.google.common.cache.CacheLoader +com.google.common.collect.Iterables @defaultMessage Do not violate java's access system java.lang.reflect.AccessibleObject#setAccessible(boolean) -java.lang.reflect.AccessibleObject#setAccessible(java.lang.reflect.AccessibleObject[], boolean) \ No newline at end of file +java.lang.reflect.AccessibleObject#setAccessible(java.lang.reflect.AccessibleObject[], boolean) From ff4a11aa32b43a1109a364b5d4fed4f8310d0f6c Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 14 Sep 2015 22:45:12 +0200 Subject: [PATCH 13/33] Replace and ban next batch of Guava classes This commit replaces and bans: * com.google.common.util.concurrent.UncheckedExecutionException * com.google.common.util.concurrent.AtomicLongMap * com.google.common.primitives.Longs * com.google.common.io.ByteStreams * com.google.common.collect.UnmodifiableIterator * com.google.common.collect.ObjectArrays * com.google.common.collect.Multimap * com.google.common.collect.MultimapBuilder Relates to #13224 --- .../lucene/queries/BlendedTermQuery.java | 4 +- .../get/GetRepositoriesRequestBuilder.java | 4 +- .../get/GetSnapshotsRequestBuilder.java | 4 +- .../status/SnapshotsStatusRequestBuilder.java | 5 +- .../alias/get/BaseAliasesRequestBuilder.java | 7 +-- .../admin/indices/get/GetIndexRequest.java | 5 +- .../get/GetFieldMappingsRequestBuilder.java | 8 +-- .../get/GetSettingsRequestBuilder.java | 4 +- .../warmer/get/GetWarmersRequestBuilder.java | 4 +- .../info/ClusterInfoRequestBuilder.java | 8 +-- .../action/termvectors/TermVectorsFilter.java | 7 ++- .../cluster/metadata/AliasOrIndex.java | 8 ++- .../cluster/metadata/MetaData.java | 16 +---- .../cluster/node/DiscoveryNodes.java | 9 +-- .../cluster/routing/IndexRoutingTable.java | 10 +-- .../cluster/routing/RoutingTable.java | 3 +- .../common/cli/CliToolConfig.java | 13 ++-- .../common/collect/CopyOnWriteHashMap.java | 9 ++- .../common/collect/ImmutableOpenIntMap.java | 19 ++++-- .../common/collect/ImmutableOpenMap.java | 27 ++++---- .../common/lucene/ShardCoreKeyMap.java | 63 ++++++++++++------- .../elasticsearch/common/util/ArrayUtils.java | 19 ++++++ .../common/util/LongObjectPagedHashMap.java | 8 ++- .../org/elasticsearch/http/HttpServer.java | 8 ++- .../internal/FieldNamesFieldMapper.java | 8 ++- .../random/RandomScoreFunctionParser.java | 10 +-- .../blobstore/BlobStoreRepository.java | 8 ++- .../blobstore/ChecksumBlobStoreFormat.java | 6 +- .../blobstore/LegacyBlobStoreFormat.java | 8 ++- .../bucket/terms/InternalTerms.java | 15 ++--- .../hdr/InternalHDRPercentileRanks.java | 9 ++- .../hdr/InternalHDRPercentiles.java | 9 ++- .../InternalTDigestPercentileRanks.java | 9 ++- .../tdigest/InternalTDigestPercentiles.java | 9 ++- .../percentile/InternalPercentilesBucket.java | 8 ++- .../search/internal/SearchContext.java | 25 ++++---- .../common/util/ArrayUtilsTests.java | 21 +++++++ .../plugins/PluginManagerUnitTests.java | 8 +-- .../resources/forbidden/all-signatures.txt | 8 +++ 39 files changed, 263 insertions(+), 170 deletions(-) diff --git a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java index 7d3c47f9615..e411139bab0 100644 --- a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java @@ -18,8 +18,6 @@ */ package org.apache.lucene.queries; -import com.google.common.primitives.Ints; - import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; @@ -141,7 +139,7 @@ public abstract class BlendedTermQuery extends Query { } @Override protected int compare(int i, int j) { - return Ints.compare(contexts[tieBreak[j]].docFreq(), contexts[tieBreak[i]].docFreq()); + return Integer.compare(contexts[tieBreak[j]].docFreq(), contexts[tieBreak[i]].docFreq()); } }.sort(0, tieBreak.length); int prev = contexts[tieBreak[0]].docFreq(); diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java index 320794c5660..e21aa19f7f8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesRequestBuilder.java @@ -19,9 +19,9 @@ package org.elasticsearch.action.admin.cluster.repositories.get; -import com.google.common.collect.ObjectArrays; import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.util.ArrayUtils; /** * Get repository request builder @@ -60,7 +60,7 @@ public class GetRepositoriesRequestBuilder extends MasterNodeReadOperationReques * @return builder */ public GetRepositoriesRequestBuilder addRepositories(String... repositories) { - request.repositories(ObjectArrays.concat(request.repositories(), repositories, String.class)); + request.repositories(ArrayUtils.concat(request.repositories(), repositories)); return this; } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java index 79472d4313a..d989ac3d95c 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java @@ -19,9 +19,9 @@ package org.elasticsearch.action.admin.cluster.snapshots.get; -import com.google.common.collect.ObjectArrays; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.util.ArrayUtils; /** * Get snapshots request builder @@ -81,7 +81,7 @@ public class GetSnapshotsRequestBuilder extends MasterNodeOperationRequestBuilde * @return this builder */ public GetSnapshotsRequestBuilder addSnapshots(String... snapshots) { - request.snapshots(ObjectArrays.concat(request.snapshots(), snapshots, String.class)); + request.snapshots(ArrayUtils.concat(request.snapshots(), snapshots)); return this; } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java index c8a7829c483..3ec1733ceab 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java @@ -19,10 +19,9 @@ package org.elasticsearch.action.admin.cluster.snapshots.status; -import com.google.common.collect.ObjectArrays; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.util.ArrayUtils; /** * Snapshots status request builder @@ -72,7 +71,7 @@ public class SnapshotsStatusRequestBuilder extends MasterNodeOperationRequestBui * @return this builder */ public SnapshotsStatusRequestBuilder addSnapshots(String... snapshots) { - request.snapshots(ObjectArrays.concat(request.snapshots(), snapshots, String.class)); + request.snapshots(ArrayUtils.concat(request.snapshots(), snapshots)); return this; } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java index da7c505771c..bcfdb0e10d8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java @@ -19,13 +19,12 @@ package org.elasticsearch.action.admin.indices.alias.get; -import com.google.common.collect.ObjectArrays; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.common.util.ArrayUtils; /** */ @@ -43,7 +42,7 @@ public abstract class BaseAliasesRequestBuilder { if (this.features == DEFAULT_FEATURES) { return features(features); } else { - return features(ObjectArrays.concat(featuresAsEnums(), features, Feature.class)); + return features(ArrayUtils.concat(features(), features, Feature.class)); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java index 158e2db4eed..84b16ac2039 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java @@ -19,12 +19,10 @@ package org.elasticsearch.action.admin.indices.mapping.get; -import com.google.common.collect.ObjectArrays; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.common.util.ArrayUtils; /** A helper class to build {@link GetFieldMappingsRequest} objects */ public class GetFieldMappingsRequestBuilder extends ActionRequestBuilder { @@ -39,7 +37,7 @@ public class GetFieldMappingsRequestBuilder extends ActionRequestBuilder selectedFields; private AggregatedDfs dfs; private Map scoreTerms; - private AtomicLongMap sizes; + private Map sizes = new HashMap<>(); private TFIDFSimilarity similarity; public TermVectorsFilter(Fields termVectorsByField, Fields topLevelFields, Set selectedFields, @Nullable AggregatedDfs dfs) { @@ -64,7 +64,6 @@ public class TermVectorsFilter { this.dfs = dfs; this.scoreTerms = new HashMap<>(); - this.sizes = AtomicLongMap.create(); this.similarity = new DefaultSimilarity(); } @@ -228,10 +227,12 @@ public class TermVectorsFilter { // retain the best terms for quick lookups ScoreTerm scoreTerm; + int count = 0; while ((scoreTerm = queue.pop()) != null) { scoreTerms.put(new Term(scoreTerm.field, scoreTerm.word), scoreTerm); - sizes.incrementAndGet(scoreTerm.field); + count++; } + sizes.put(fieldName, count); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java index e98a046238d..b8de2ea5256 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java @@ -19,7 +19,6 @@ package org.elasticsearch.cluster.metadata; -import com.google.common.collect.UnmodifiableIterator; import org.elasticsearch.common.collect.Tuple; import java.util.ArrayList; @@ -106,7 +105,7 @@ public interface AliasOrIndex { return new Iterable>() { @Override public Iterator> iterator() { - return new UnmodifiableIterator>() { + return new Iterator>() { int index = 0; @@ -121,6 +120,11 @@ public interface AliasOrIndex { return new Tuple<>(indexMetaData.getIndex(), indexMetaData.getAliases().get(aliasName)); } + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } + }; } }; diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 8ce7f82f323..3ca3ec39bbd 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -23,7 +23,6 @@ import com.carrotsearch.hppc.ObjectHashSet; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.UnmodifiableIterator; import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.Diffable; @@ -60,18 +59,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; +import java.util.*; import java.util.stream.Collectors; import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; @@ -569,7 +557,7 @@ public class MetaData implements Iterable, Diffable, Fr } @Override - public UnmodifiableIterator iterator() { + public Iterator iterator() { return indices.valuesIt(); } diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index 847173ec1a7..13b64716336 100644 --- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -22,7 +22,6 @@ package org.elasticsearch.cluster.node; import com.carrotsearch.hppc.ObjectHashSet; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import com.google.common.collect.UnmodifiableIterator; import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.common.Booleans; @@ -34,11 +33,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.transport.TransportAddress; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * This class holds all {@link DiscoveryNode} in the cluster and provides convenience methods to @@ -69,7 +64,7 @@ public class DiscoveryNodes extends AbstractDiffable implements } @Override - public UnmodifiableIterator iterator() { + public Iterator iterator() { return nodes.valuesIt(); } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index a909b5550fe..062c7b4035d 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -22,7 +22,6 @@ package org.elasticsearch.cluster.routing; import com.carrotsearch.hppc.IntSet; import com.carrotsearch.hppc.cursors.IntCursor; import com.carrotsearch.hppc.cursors.IntObjectCursor; -import com.google.common.collect.UnmodifiableIterator; import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -33,12 +32,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.shard.ShardId; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.concurrent.ThreadLocalRandom; /** @@ -163,7 +157,7 @@ public class IndexRoutingTable extends AbstractDiffable imple } @Override - public UnmodifiableIterator iterator() { + public Iterator iterator() { return shards.valuesIt(); } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index 07fe33acad9..32c50b3fd1f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -21,7 +21,6 @@ package org.elasticsearch.cluster.routing; import com.carrotsearch.hppc.IntSet; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.UnmodifiableIterator; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.Diffable; import org.elasticsearch.cluster.DiffableUtils; @@ -68,7 +67,7 @@ public class RoutingTable implements Iterable, Diffable iterator() { + public Iterator iterator() { return indicesRouting.values().iterator(); } diff --git a/core/src/main/java/org/elasticsearch/common/cli/CliToolConfig.java b/core/src/main/java/org/elasticsearch/common/cli/CliToolConfig.java index 2e3d755793e..d0ba897b33d 100644 --- a/core/src/main/java/org/elasticsearch/common/cli/CliToolConfig.java +++ b/core/src/main/java/org/elasticsearch/common/cli/CliToolConfig.java @@ -19,13 +19,14 @@ package org.elasticsearch.common.cli; -import com.google.common.collect.ImmutableMap; -import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionGroup; import org.apache.commons.cli.Options; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; /** * @@ -38,18 +39,18 @@ public class CliToolConfig { private final Class toolType; private final String name; - private final ImmutableMap cmds; + private final Map cmds; private static final HelpPrinter helpPrinter = new HelpPrinter(); private CliToolConfig(String name, Class toolType, Cmd[] cmds) { this.name = name; this.toolType = toolType; - ImmutableMap.Builder cmdsBuilder = ImmutableMap.builder(); + final Map cmdsMapping = new HashMap<>(); for (int i = 0; i < cmds.length; i++) { - cmdsBuilder.put(cmds[i].name, cmds[i]); + cmdsMapping.put(cmds[i].name, cmds[i]); } - this.cmds = cmdsBuilder.build(); + this.cmds = Collections.unmodifiableMap(cmdsMapping); } public boolean isSingle() { diff --git a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java index eab20aaa23b..c0114480498 100644 --- a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java +++ b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java @@ -18,8 +18,6 @@ */ package org.elasticsearch.common.collect; - -import com.google.common.collect.UnmodifiableIterator; import org.apache.lucene.util.mutable.MutableValueInt; import java.lang.reflect.Array; @@ -420,7 +418,7 @@ public final class CopyOnWriteHashMap extends AbstractMap { } - private static class EntryIterator extends UnmodifiableIterator> { + private static class EntryIterator implements Iterator> { private final Deque> entries; private final Deque> nodes; @@ -448,6 +446,11 @@ public final class CopyOnWriteHashMap extends AbstractMap { return entries.pop(); } + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } + } private final InnerNode root; diff --git a/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenIntMap.java b/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenIntMap.java index eda83a1505b..ccadbcf0a4a 100644 --- a/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenIntMap.java +++ b/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenIntMap.java @@ -26,7 +26,6 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.predicates.IntObjectPredicate; import com.carrotsearch.hppc.predicates.IntPredicate; import com.carrotsearch.hppc.procedures.IntObjectProcedure; -import com.google.common.collect.UnmodifiableIterator; import java.util.Iterator; import java.util.Map; @@ -113,9 +112,9 @@ public final class ImmutableOpenIntMap implements Iterable keysIt() { + public Iterator keysIt() { final Iterator iterator = map.keys().iterator(); - return new UnmodifiableIterator() { + return new Iterator() { @Override public boolean hasNext() { return iterator.hasNext(); @@ -125,6 +124,11 @@ public final class ImmutableOpenIntMap implements Iterable implements Iterable valuesIt() { + public Iterator valuesIt() { final Iterator> iterator = map.values().iterator(); - return new UnmodifiableIterator() { + return new Iterator() { @Override public boolean hasNext() { return iterator.hasNext(); @@ -150,6 +154,11 @@ public final class ImmutableOpenIntMap implements Iterable implements Iterable keysIt() { + public Iterator keysIt() { final Iterator> iterator = map.keys().iterator(); - return new UnmodifiableIterator() { + return new Iterator() { @Override - public boolean hasNext() { - return iterator.hasNext(); - } + public boolean hasNext() { return iterator.hasNext(); } @Override public KType next() { return iterator.next().value; } + + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } }; } @@ -145,18 +147,21 @@ public final class ImmutableOpenMap implements Iterable valuesIt() { + public Iterator valuesIt() { final Iterator> iterator = map.values().iterator(); - return new UnmodifiableIterator() { + return new Iterator() { @Override - public boolean hasNext() { - return iterator.hasNext(); - } + public boolean hasNext() { return iterator.hasNext(); } @Override public VType next() { return iterator.next().value; } + + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } }; } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java index 0d9270edaff..c405de129a5 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java @@ -19,19 +19,11 @@ package org.elasticsearch.common.lucene; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Multimap; - import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReader.CoreClosedListener; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; -import java.io.IOException; -import java.util.IdentityHashMap; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * A map between segment core cache keys and the shard that these segments @@ -47,16 +39,16 @@ import java.util.Set; public final class ShardCoreKeyMap { private final Map coreKeyToShard; - private final Multimap indexToCoreKey; + private final Map> indexToCoreKey; public ShardCoreKeyMap() { coreKeyToShard = new IdentityHashMap<>(); - indexToCoreKey = HashMultimap.create(); + indexToCoreKey = new HashMap<>(); } /** * Register a {@link LeafReader}. This is necessary so that the core cache - * key of this reader can be found later using {@link #getCoreCacheKeys(ShardId)}. + * key of this reader can be found later using {@link #getCoreKeysForIndex(String)}. */ public void add(LeafReader reader) { final ShardId shardId = ShardUtils.extractShardId(reader); @@ -67,15 +59,22 @@ public final class ShardCoreKeyMap { final String index = shardId.getIndex(); synchronized (this) { if (coreKeyToShard.put(coreKey, shardId) == null) { - final boolean added = indexToCoreKey.put(index, coreKey); + Set objects = indexToCoreKey.get(index); + if (objects == null) { + objects = new HashSet<>(); + indexToCoreKey.put(index, objects); + } + final boolean added = objects.add(coreKey); assert added; - reader.addCoreClosedListener(new CoreClosedListener() { - @Override - public void onClose(Object ownerCoreCacheKey) throws IOException { - assert coreKey == ownerCoreCacheKey; - synchronized (ShardCoreKeyMap.this) { - coreKeyToShard.remove(ownerCoreCacheKey); - indexToCoreKey.remove(index, coreKey); + reader.addCoreClosedListener(ownerCoreCacheKey -> { + assert coreKey == ownerCoreCacheKey; + synchronized (ShardCoreKeyMap.this) { + coreKeyToShard.remove(ownerCoreCacheKey); + final Set coreKeys = indexToCoreKey.get(index); + final boolean removed = coreKeys.remove(coreKey); + assert removed; + if (coreKeys.isEmpty()) { + indexToCoreKey.remove(index); } } }); @@ -95,15 +94,35 @@ public final class ShardCoreKeyMap { * Get the set of core cache keys associated with the given index. */ public synchronized Set getCoreKeysForIndex(String index) { - return ImmutableSet.copyOf(indexToCoreKey.get(index)); + final Set objects = indexToCoreKey.get(index); + if (objects == null) { + return Collections.emptySet(); + } + // we have to copy otherwise we risk ConcurrentModificationException + return Collections.unmodifiableSet(new HashSet<>(objects)); } /** * Return the number of tracked segments. */ public synchronized int size() { - assert indexToCoreKey.size() == coreKeyToShard.size(); + assert assertSize(); return coreKeyToShard.size(); } + private synchronized boolean assertSize() { + // this is heavy and should only used in assertions + boolean assertionsEnabled = false; + assert assertionsEnabled = true; + if (assertionsEnabled == false) { + throw new AssertionError("only run this if assertions are enabled"); + } + Collection> values = indexToCoreKey.values(); + int size = 0; + for (Set value : values) { + size += value.size(); + } + return size == coreKeyToShard.size(); + } + } diff --git a/core/src/main/java/org/elasticsearch/common/util/ArrayUtils.java b/core/src/main/java/org/elasticsearch/common/util/ArrayUtils.java index 053c9b547ae..bb8442efa08 100644 --- a/core/src/main/java/org/elasticsearch/common/util/ArrayUtils.java +++ b/core/src/main/java/org/elasticsearch/common/util/ArrayUtils.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.util; +import java.lang.reflect.Array; import java.util.Arrays; /** @@ -69,4 +70,22 @@ public class ArrayUtils { } return index; } + + /** + * Concatenates 2 arrays + */ + public static String[] concat(String[] one, String[] other) { + return concat(one, other, String.class); + } + + /** + * Concatenates 2 arrays + */ + public static T[] concat(T[] one, T[] other, Class clazz) { + T[] target = (T[]) Array.newInstance(clazz, one.length + other.length); + System.arraycopy(one, 0, target, 0, one.length); + System.arraycopy(other, 0, target, one.length, other.length); + return target; + } + } diff --git a/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java b/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java index 7949989c2a0..63b7b23a62b 100644 --- a/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java +++ b/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java @@ -20,7 +20,6 @@ package org.elasticsearch.common.util; -import com.google.common.collect.UnmodifiableIterator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lease.Releasables; @@ -125,7 +124,7 @@ public class LongObjectPagedHashMap extends AbstractPagedHashMap implements I @Override public Iterator> iterator() { - return new UnmodifiableIterator>() { + return new Iterator>() { boolean cached; final Cursor cursor; @@ -162,6 +161,11 @@ public class LongObjectPagedHashMap extends AbstractPagedHashMap implements I return cursor; } + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } + }; } diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java index 0fab142ac18..168ed0669e8 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpServer.java +++ b/core/src/main/java/org/elasticsearch/http/HttpServer.java @@ -20,16 +20,17 @@ package org.elasticsearch.http; import com.google.common.collect.ImmutableMap; -import com.google.common.io.ByteStreams; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.node.service.NodeService; import org.elasticsearch.rest.*; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.*; @@ -141,8 +142,9 @@ public class HttpServer extends AbstractLifecycleComponent { if (request.method() == RestRequest.Method.GET) { try { try (InputStream stream = getClass().getResourceAsStream("/config/favicon.ico")) { - byte[] content = ByteStreams.toByteArray(stream); - BytesRestResponse restResponse = new BytesRestResponse(RestStatus.OK, "image/x-icon", content); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Streams.copy(stream, out); + BytesRestResponse restResponse = new BytesRestResponse(RestStatus.OK, "image/x-icon", out.toByteArray()); channel.sendResponse(restResponse); } } catch (IOException e) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java index ac2ef99c20b..500f973e0ea 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper.internal; -import com.google.common.collect.UnmodifiableIterator; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -240,7 +239,7 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { return new Iterable() { @Override public Iterator iterator() { - return new UnmodifiableIterator() { + return new Iterator() { int endIndex = nextEndIndex(0); @@ -263,6 +262,11 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { return result; } + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } + }; } }; diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/random/RandomScoreFunctionParser.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/random/RandomScoreFunctionParser.java index 124336c5bbe..6d8f72982f3 100644 --- a/core/src/main/java/org/elasticsearch/index/query/functionscore/random/RandomScoreFunctionParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/random/RandomScoreFunctionParser.java @@ -20,14 +20,12 @@ package org.elasticsearch.index.query.functionscore.random; -import com.google.common.primitives.Longs; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.function.RandomScoreFunction; import org.elasticsearch.common.lucene.search.function.ScoreFunction; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryParsingException; @@ -66,7 +64,7 @@ public class RandomScoreFunctionParser implements ScoreFunctionParser { if (parser.numberType() == XContentParser.NumberType.INT) { seed = parser.intValue(); } else if (parser.numberType() == XContentParser.NumberType.LONG) { - seed = Longs.hashCode(parser.longValue()); + seed = hash(parser.longValue()); } else { throw new QueryParsingException(parseContext, "random_score seed must be an int, long or string, not '" + token.toString() + "'"); @@ -90,7 +88,7 @@ public class RandomScoreFunctionParser implements ScoreFunctionParser { } if (seed == -1) { - seed = Longs.hashCode(parseContext.nowInMillis()); + seed = hash(parseContext.nowInMillis()); } final ShardId shardId = SearchContext.current().indexShard().shardId(); final int salt = (shardId.index().name().hashCode() << 10) | shardId.id(); @@ -98,4 +96,8 @@ public class RandomScoreFunctionParser implements ScoreFunctionParser { return new RandomScoreFunction(seed, salt, uidFieldData); } + + private static final int hash(long value) { + return (int) (value ^ (value >>> 32)); + } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 4b2c608f1f4..456457861df 100644 --- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -19,7 +19,6 @@ package org.elasticsearch.repositories.blobstore; -import com.google.common.io.ByteStreams; import org.apache.lucene.store.RateLimiter; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; @@ -36,6 +35,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.compress.NotXContentException; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -66,6 +66,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.util.ArrayList; import java.util.Collections; @@ -590,9 +591,10 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent readSnapshotList() throws IOException { try (InputStream blob = snapshotsBlobContainer.readBlob(SNAPSHOTS_FILE)) { - final byte[] data = ByteStreams.toByteArray(blob); + BytesStreamOutput out = new BytesStreamOutput(); + Streams.copy(blob, out); ArrayList snapshots = new ArrayList<>(); - try (XContentParser parser = XContentHelper.createParser(new BytesArray(data))) { + try (XContentParser parser = XContentHelper.createParser(out.bytes())) { if (parser.nextToken() == XContentParser.Token.START_OBJECT) { if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { String currentFieldName = parser.currentName(); diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java index 718e9da7337..9109dcd84bc 100644 --- a/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java +++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.repositories.blobstore; -import com.google.common.io.ByteStreams; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; @@ -29,6 +28,7 @@ import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressorFactory; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.store.ByteArrayIndexInput; @@ -93,7 +93,9 @@ public class ChecksumBlobStoreFormat extends BlobStoreForm */ public T readBlob(BlobContainer blobContainer, String blobName) throws IOException { try (InputStream inputStream = blobContainer.readBlob(blobName)) { - byte[] bytes = ByteStreams.toByteArray(inputStream); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Streams.copy(inputStream, out); + final byte[] bytes = out.toByteArray(); final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")"; try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) { CodecUtil.checksumEntireFile(indexInput); diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/LegacyBlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/LegacyBlobStoreFormat.java index f9d5c98253c..a0c956dea45 100644 --- a/core/src/main/java/org/elasticsearch/repositories/blobstore/LegacyBlobStoreFormat.java +++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/LegacyBlobStoreFormat.java @@ -18,10 +18,10 @@ */ package org.elasticsearch.repositories.blobstore; -import com.google.common.io.ByteStreams; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.blobstore.BlobContainer; -import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.FromXContentBuilder; import org.elasticsearch.common.xcontent.ToXContent; @@ -53,7 +53,9 @@ public class LegacyBlobStoreFormat extends BlobStoreFormat */ public T readBlob(BlobContainer blobContainer, String blobName) throws IOException { try (InputStream inputStream = blobContainer.readBlob(blobName)) { - return read(new BytesArray(ByteStreams.toByteArray(inputStream))); + BytesStreamOutput out = new BytesStreamOutput(); + Streams.copy(inputStream, out); + return read(out.bytes()); } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java index 681d18651d8..31285d20f8e 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java @@ -18,8 +18,6 @@ */ package org.elasticsearch.search.aggregations.bucket.terms; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.search.aggregations.AggregationExecutionException; @@ -33,7 +31,6 @@ import org.elasticsearch.search.aggregations.support.format.ValueFormatter; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -167,7 +164,7 @@ public abstract class InternalTerms aggregations, ReduceContext reduceContext) { - Multimap buckets = ArrayListMultimap.create(); + Map> buckets = new HashMap<>(); long sumDocCountError = 0; long otherDocCount = 0; InternalTerms referenceTerms = null; @@ -208,14 +205,18 @@ public abstract class InternalTerms bucketList = buckets.get(bucket.getKey()); + if (bucketList == null) { + bucketList = new ArrayList<>(); + buckets.put(bucket.getKey(), bucketList); + } + bucketList.add(bucket); } } final int size = Math.min(requiredSize, buckets.size()); BucketPriorityQueue ordered = new BucketPriorityQueue(size, order.comparator(null)); - for (Collection l : buckets.asMap().values()) { - List sameTermBuckets = (List) l; // cast is ok according to javadocs + for (List sameTermBuckets : buckets.values()) { final Bucket b = sameTermBuckets.get(0).reduce(sameTermBuckets, reduceContext); if (b.docCountError != -1) { if (sumDocCountError == -1) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java index 37acb5313b5..baa2325a734 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java @@ -18,8 +18,6 @@ */ package org.elasticsearch.search.aggregations.metrics.percentiles.hdr; -import com.google.common.collect.UnmodifiableIterator; - import org.HdrHistogram.DoubleHistogram; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.search.aggregations.AggregationStreams; @@ -106,7 +104,7 @@ public class InternalHDRPercentileRanks extends AbstractInternalHDRPercentiles i return percentileRank; } - public static class Iter extends UnmodifiableIterator { + public static class Iter implements Iterator { private final double[] values; private final DoubleHistogram state; @@ -129,5 +127,10 @@ public class InternalHDRPercentileRanks extends AbstractInternalHDRPercentiles i ++i; return next; } + + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java index 4c475ed46a8..b7b126f0a6b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java @@ -18,8 +18,6 @@ */ package org.elasticsearch.search.aggregations.metrics.percentiles.hdr; -import com.google.common.collect.UnmodifiableIterator; - import org.HdrHistogram.DoubleHistogram; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.search.aggregations.AggregationStreams; @@ -96,7 +94,7 @@ public class InternalHDRPercentiles extends AbstractInternalHDRPercentiles imple return TYPE; } - public static class Iter extends UnmodifiableIterator { + public static class Iter implements Iterator { private final double[] percents; private final DoubleHistogram state; @@ -119,5 +117,10 @@ public class InternalHDRPercentiles extends AbstractInternalHDRPercentiles imple ++i; return next; } + + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java index bff0dcf2664..61deecb279d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java @@ -18,8 +18,6 @@ */ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest; -import com.google.common.collect.UnmodifiableIterator; - import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.search.aggregations.AggregationStreams; import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile; @@ -102,7 +100,7 @@ public class InternalTDigestPercentileRanks extends AbstractInternalTDigestPerce return percentileRank * 100; } - public static class Iter extends UnmodifiableIterator { + public static class Iter implements Iterator { private final double[] values; private final TDigestState state; @@ -125,5 +123,10 @@ public class InternalTDigestPercentileRanks extends AbstractInternalTDigestPerce ++i; return next; } + + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java index 564e9d538ef..878ef559ba4 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java @@ -18,8 +18,6 @@ */ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest; -import com.google.common.collect.UnmodifiableIterator; - import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.search.aggregations.AggregationStreams; import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile; @@ -92,7 +90,7 @@ public class InternalTDigestPercentiles extends AbstractInternalTDigestPercentil return TYPE; } - public static class Iter extends UnmodifiableIterator { + public static class Iter implements Iterator { private final double[] percents; private final TDigestState state; @@ -115,5 +113,10 @@ public class InternalTDigestPercentiles extends AbstractInternalTDigestPercentil ++i; return next; } + + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java index 10b1481e8d6..6a11c6fafd7 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile; -import com.google.common.collect.UnmodifiableIterator; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -136,7 +135,7 @@ public class InternalPercentilesBucket extends InternalNumericMetricsAggregation return builder; } - public static class Iter extends UnmodifiableIterator { + public static class Iter implements Iterator { private final double[] percents; private final double[] percentiles; @@ -159,5 +158,10 @@ public class InternalPercentilesBucket extends InternalNumericMetricsAggregation ++i; return next; } + + @Override + public final void remove() { + throw new UnsupportedOperationException(); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 34f1d5a8f45..6402d5e53d8 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -18,8 +18,6 @@ */ package org.elasticsearch.search.internal; -import com.google.common.collect.Multimap; -import com.google.common.collect.MultimapBuilder; import org.apache.lucene.search.Collector; import org.apache.lucene.search.Query; @@ -62,10 +60,7 @@ import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rescore.RescoreSearchContext; import org.elasticsearch.search.suggest.SuggestionSearchContext; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; public abstract class SearchContext extends DelegatingHasContextAndHeaders implements Releasable { @@ -87,7 +82,7 @@ public abstract class SearchContext extends DelegatingHasContextAndHeaders imple return current.get(); } - private Multimap clearables = null; + private Map> clearables = null; private final AtomicBoolean closed = new AtomicBoolean(false); protected final ParseFieldMatcher parseFieldMatcher; @@ -316,19 +311,27 @@ public abstract class SearchContext extends DelegatingHasContextAndHeaders imple */ public void addReleasable(Releasable releasable, Lifetime lifetime) { if (clearables == null) { - clearables = MultimapBuilder.enumKeys(Lifetime.class).arrayListValues().build(); + clearables = new HashMap<>(); } - clearables.put(lifetime, releasable); + List releasables = clearables.get(lifetime); + if (releasables == null) { + releasables = new ArrayList<>(); + clearables.put(lifetime, releasables); + } + releasables.add(releasable); } public void clearReleasables(Lifetime lifetime) { if (clearables != null) { - List> releasables = new ArrayList<>(); + List>releasables = new ArrayList<>(); for (Lifetime lc : Lifetime.values()) { if (lc.compareTo(lifetime) > 0) { break; } - releasables.add(clearables.removeAll(lc)); + List remove = clearables.remove(lc); + if (remove != null) { + releasables.add(remove); + } } Releasables.close(Iterables.flatten(releasables)); } diff --git a/core/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java b/core/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java index 0c7038e8cd1..735fda16797 100644 --- a/core/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java @@ -19,9 +19,12 @@ package org.elasticsearch.common.util; +import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.test.ESTestCase; import org.junit.Test; +import java.lang.reflect.Array; +import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -90,4 +93,22 @@ public class ArrayUtilsTests extends ESTestCase { return min + delta; } + public void testConcat() { + assertArrayEquals(new String[]{"a", "b", "c", "d"}, ArrayUtils.concat(new String[]{"a", "b"}, new String[]{"c", "d"})); + int firstSize = randomIntBetween(0, 10); + String[] first = new String[firstSize]; + ArrayList sourceOfTruth = new ArrayList<>(); + for (int i = 0; i < firstSize; i++) { + first[i] = randomRealisticUnicodeOfCodepointLengthBetween(0,10); + sourceOfTruth.add(first[i]); + } + int secondSize = randomIntBetween(0, 10); + String[] second = new String[secondSize]; + for (int i = 0; i < secondSize; i++) { + second[i] = randomRealisticUnicodeOfCodepointLengthBetween(0, 10); + sourceOfTruth.add(second[i]); + } + assertArrayEquals(sourceOfTruth.toArray(new String[0]), ArrayUtils.concat(first, second)); + } + } diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginManagerUnitTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginManagerUnitTests.java index cf44a7614dc..c1ae4c22ea2 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginManagerUnitTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginManagerUnitTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.plugins; -import com.google.common.io.Files; import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.common.http.client.HttpDownloadHelper; @@ -63,10 +62,9 @@ public class PluginManagerUnitTests extends ESTestCase { Environment environment = new Environment(settings); PluginManager.PluginHandle pluginHandle = new PluginManager.PluginHandle(pluginName, "version", "user"); - String configDirPath = Files.simplifyPath(pluginHandle.configDir(environment).normalize().toString()); - String expectedDirPath = Files.simplifyPath(genericConfigFolder.resolve(pluginName).normalize().toString()); - - assertThat(configDirPath, is(expectedDirPath)); + Path configDirPath = pluginHandle.configDir(environment).normalize(); + Path expectedDirPath = genericConfigFolder.resolve(pluginName).normalize(); + assertEquals(configDirPath, expectedDirPath); } @Test diff --git a/dev-tools/src/main/resources/forbidden/all-signatures.txt b/dev-tools/src/main/resources/forbidden/all-signatures.txt index 82237cce048..c79dc5c7d26 100644 --- a/dev-tools/src/main/resources/forbidden/all-signatures.txt +++ b/dev-tools/src/main/resources/forbidden/all-signatures.txt @@ -111,6 +111,14 @@ com.google.common.collect.Collections2 com.google.common.cache.LoadingCache com.google.common.cache.CacheLoader com.google.common.collect.Iterables +com.google.common.util.concurrent.UncheckedExecutionException +com.google.common.util.concurrent.AtomicLongMap +com.google.common.primitives.Longs +com.google.common.io.ByteStreams +com.google.common.collect.UnmodifiableIterator +com.google.common.collect.ObjectArrays +com.google.common.collect.Multimap +com.google.common.collect.MultimapBuilder @defaultMessage Do not violate java's access system java.lang.reflect.AccessibleObject#setAccessible(boolean) From 7a29febd721b74c58cb480c601f3c3660841b200 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 15 Sep 2015 10:07:15 -0400 Subject: [PATCH 14/33] Remove and forbid use of com.google.common.math.LongMath This commit removes and now forbids all uses of com.google.common.math.LongMath across the codebase. This is one step of many in the eventual removal of Guava as a dependency. --- .../org/elasticsearch/common/util/BloomFilter.java | 10 +++++++--- .../src/main/resources/forbidden/all-signatures.txt | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/util/BloomFilter.java b/core/src/main/java/org/elasticsearch/common/util/BloomFilter.java index 8a2acba8d28..0f72d6d62b1 100644 --- a/core/src/main/java/org/elasticsearch/common/util/BloomFilter.java +++ b/core/src/main/java/org/elasticsearch/common/util/BloomFilter.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.common.util; -import com.google.common.math.LongMath; import com.google.common.primitives.Ints; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.DataOutput; @@ -33,7 +32,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.SizeValue; import java.io.IOException; -import java.math.RoundingMode; import java.util.Arrays; import java.util.Comparator; @@ -321,7 +319,13 @@ public class BloomFilter { long bitCount; BitArray(long bits) { - this(new long[Ints.checkedCast(LongMath.divide(bits, 64, RoundingMode.CEILING))]); + this(new long[size(bits)]); + } + + private static int size(long bits) { + long quotient = bits / 64; + long remainder = bits - quotient * 64; + return Ints.checkedCast(remainder == 0 ? quotient : 1 + quotient); } // Used by serialization diff --git a/dev-tools/src/main/resources/forbidden/all-signatures.txt b/dev-tools/src/main/resources/forbidden/all-signatures.txt index c79dc5c7d26..c133fd51004 100644 --- a/dev-tools/src/main/resources/forbidden/all-signatures.txt +++ b/dev-tools/src/main/resources/forbidden/all-signatures.txt @@ -119,6 +119,7 @@ com.google.common.collect.UnmodifiableIterator com.google.common.collect.ObjectArrays com.google.common.collect.Multimap com.google.common.collect.MultimapBuilder +com.google.common.math.LongMath @defaultMessage Do not violate java's access system java.lang.reflect.AccessibleObject#setAccessible(boolean) From b3c6327caf8cba43db2f9f294713d58a56df55ec Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 15 Sep 2015 09:10:05 -0400 Subject: [PATCH 15/33] Remove and forbid use of com.google.common.base.Joiner This commit removes and now forbids all uses of com.google.common.base.Joiner across the codebase. This is one of many steps in the eventual removal of Guava as a dependency. Relates #13224 --- .../common/util/concurrent/EsExecutors.java | 10 +- .../repositories/RepositoriesService.java | 20 +-- .../context/CategoryContextMapping.java | 11 +- .../elasticsearch/NamingConventionTests.java | 43 +++--- .../elasticsearch/test/ESIntegTestCase.java | 2 +- .../rest/client/http/HttpRequestBuilder.java | 12 +- .../test/rest/section/ApiCallSection.java | 9 +- .../resources/forbidden/#all-signatures.txt# | 129 ++++++++++++++++++ .../resources/forbidden/all-signatures.txt | 1 + 9 files changed, 181 insertions(+), 56 deletions(-) create mode 100644 dev-tools/src/main/resources/forbidden/#all-signatures.txt# diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java index c7cc07c3d45..46fd7507f8b 100644 --- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java +++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java @@ -19,11 +19,12 @@ package org.elasticsearch.common.util.concurrent; -import com.google.common.base.Joiner; import org.elasticsearch.common.settings.Settings; +import java.util.Arrays; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; /** * @@ -81,7 +82,12 @@ public class EsExecutors { } public static String threadName(Settings settings, String ... names) { - return threadName(settings, "[" + Joiner.on(".").skipNulls().join(names) + "]"); + String namePrefix = + Arrays + .stream(names) + .filter(name -> name != null) + .collect(Collectors.joining(".", "[", "]")); + return threadName(settings, namePrefix); } public static String threadName(Settings settings, String namePrefix) { diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index 0a129bd8144..b11f91fa7f0 100644 --- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -19,14 +19,9 @@ package org.elasticsearch.repositories; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.AckedClusterStateUpdateTask; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.metadata.MetaData; @@ -45,10 +40,8 @@ import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.transport.TransportService; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; @@ -571,9 +564,10 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta } public String failureDescription() { - StringBuilder builder = new StringBuilder('['); - Joiner.on(", ").appendTo(builder, failures); - return builder.append(']').toString(); + return Arrays + .stream(failures) + .map(failure -> failure.toString()) + .collect(Collectors.joining(", ", "[", "]")); } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java index 34b78301ca8..ce807d824bc 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.suggest.context; -import com.google.common.base.Joiner; import org.apache.lucene.analysis.PrefixAnalyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.IndexableField; @@ -36,6 +35,8 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import java.io.IOException; import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; /** * The {@link CategoryContextMapping} is used to define a {@link ContextMapping} that @@ -258,14 +259,18 @@ public class CategoryContextMapping extends ContextMapping { public String toString() { StringBuilder sb = new StringBuilder("FieldConfig(" + fieldname + " = ["); if (this.values != null && this.values.iterator().hasNext()) { - sb.append("(").append(Joiner.on(", ").join(this.values.iterator())).append(")"); + sb.append(delimitValues(this.values)); } if (this.defaultValues != null && this.defaultValues.iterator().hasNext()) { - sb.append(" default(").append(Joiner.on(", ").join(this.defaultValues.iterator())).append(")"); + sb.append(" default").append(delimitValues(this.defaultValues)); } return sb.append("])").toString(); } + private String delimitValues(Iterable values) { + return StreamSupport.stream(values.spliterator(), false).collect(Collectors.joining(", ", "(", ")")); + } + } private static class FieldQuery extends ContextQuery { diff --git a/core/src/test/java/org/elasticsearch/NamingConventionTests.java b/core/src/test/java/org/elasticsearch/NamingConventionTests.java index 051edd5c989..40868fc281c 100644 --- a/core/src/test/java/org/elasticsearch/NamingConventionTests.java +++ b/core/src/test/java/org/elasticsearch/NamingConventionTests.java @@ -19,9 +19,6 @@ package org.elasticsearch; import junit.framework.TestCase; - -import com.google.common.base.Joiner; - import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.test.ESIntegTestCase; @@ -40,6 +37,7 @@ import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; import java.util.Set; +import java.util.stream.Collectors; /** * Simple class that ensures that all subclasses concrete of ESTestCase end with either Test | Tests @@ -149,24 +147,27 @@ public class NamingConventionTests extends ESTestCase { assertTrue(pureUnitTest.remove(PlainUnit.class)); assertTrue(pureUnitTest.remove(PlainUnitTheSecond.class)); - String classesToSubclass = Joiner.on(',').join( - ESTestCase.class.getSimpleName(), - ESTestCase.class.getSimpleName(), - ESTokenStreamTestCase.class.getSimpleName(), - LuceneTestCase.class.getSimpleName()); - assertTrue("Not all subclasses of " + ESTestCase.class.getSimpleName() + - " match the naming convention. Concrete classes must end with [Tests]:\n" + Joiner.on('\n').join(missingSuffix), - missingSuffix.isEmpty()); - assertTrue("Classes ending with [Tests] are abstract or interfaces:\n" + Joiner.on('\n').join(notRunnable), - notRunnable.isEmpty()); - assertTrue("Found inner classes that are tests, which are excluded from the test runner:\n" + Joiner.on('\n').join(innerClasses), - innerClasses.isEmpty()); - assertTrue("Pure Unit-Test found must subclass one of [" + classesToSubclass +"]:\n" + Joiner.on('\n').join(pureUnitTest), - pureUnitTest.isEmpty()); - assertTrue("Classes ending with [Tests] must subclass [" + classesToSubclass + "]:\n" + Joiner.on('\n').join(notImplementing), - notImplementing.isEmpty()); - assertTrue("Subclasses of ESIntegTestCase should end with IT as they are integration tests:\n" + Joiner.on('\n').join(integTestsInDisguise), - integTestsInDisguise.isEmpty()); + String classesToSubclass = String.join( + ",", + ESTestCase.class.getSimpleName(), + ESTestCase.class.getSimpleName(), + ESTokenStreamTestCase.class.getSimpleName(), + LuceneTestCase.class.getSimpleName() + ); + assertNoViolations("Not all subclasses of " + ESTestCase.class.getSimpleName() + " match the naming convention. Concrete classes must end with [Tests]:\n", missingSuffix); + assertNoViolations("Classes ending with [Tests] are abstract or interfaces:\n", notRunnable); + assertNoViolations("Found inner classes that are tests, which are excluded from the test runner:\n", innerClasses); + assertNoViolations("Pure Unit-Test found must subclass one of [" + classesToSubclass + "]:\n", pureUnitTest); + assertNoViolations("Classes ending with [Tests] must subclass [" + classesToSubclass + "]:\n", notImplementing); + assertNoViolations("Subclasses of ESIntegTestCase should end with IT as they are integration tests:\n", integTestsInDisguise); + } + + private String join(Set set) { + return set.stream().map(Object::toString).collect(Collectors.joining("\n")); + } + + private void assertNoViolations(String message, Set set) { + assertTrue(message + join(set), set.isEmpty()); } /* diff --git a/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java b/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java index 95c48e8f08a..59fe63a06a5 100644 --- a/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java @@ -793,7 +793,7 @@ public abstract class ESIntegTestCase extends ESTestCase { } private Settings.Builder getExcludeSettings(String index, int num, Settings.Builder builder) { - String exclude = Joiner.on(',').join(internalCluster().allDataNodesButN(num)); + String exclude = String.join(",", internalCluster().allDataNodesButN(num)); builder.put("index.routing.allocation.exclude._name", exclude); return builder; } diff --git a/core/src/test/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java b/core/src/test/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java index 3fa592b8287..2f42488a151 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java +++ b/core/src/test/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java @@ -18,14 +18,7 @@ */ package org.elasticsearch.test.rest.client.http; -import com.google.common.base.Joiner; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; -import org.apache.http.client.methods.HttpHead; -import org.apache.http.client.methods.HttpOptions; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.*; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.elasticsearch.client.support.Headers; @@ -44,6 +37,7 @@ import java.net.URLEncoder; import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; /** * Executable builder for an http request @@ -194,7 +188,7 @@ public class HttpRequestBuilder { //(e.g. '+' will stay as is) hence when trying to properly encode params manually they will end up double encoded (+ becomes %252B instead of %2B). StringBuilder uriBuilder = new StringBuilder(protocol).append("://").append(host).append(":").append(port).append(uri.getRawPath()); if (params.size() > 0) { - uriBuilder.append("?").append(Joiner.on('&').withKeyValueSeparator("=").join(params)); + uriBuilder.append("?").append(params.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&"))); } return URI.create(uriBuilder.toString()); } catch(URISyntaxException e) { diff --git a/core/src/test/java/org/elasticsearch/test/rest/section/ApiCallSection.java b/core/src/test/java/org/elasticsearch/test/rest/section/ApiCallSection.java index 29f83d1b026..852d71cc43a 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/section/ApiCallSection.java +++ b/core/src/test/java/org/elasticsearch/test/rest/section/ApiCallSection.java @@ -18,14 +18,9 @@ */ package org.elasticsearch.test.rest.section; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Represents a test fragment that contains the information needed to call an api @@ -52,7 +47,7 @@ public class ApiCallSection { public void addParam(String key, String value) { String existingValue = params.get(key); if (existingValue != null) { - value = Joiner.on(",").join(existingValue, value); + value = existingValue + "," + value; } this.params.put(key, value); } diff --git a/dev-tools/src/main/resources/forbidden/#all-signatures.txt# b/dev-tools/src/main/resources/forbidden/#all-signatures.txt# new file mode 100644 index 00000000000..c3fa24045e2 --- /dev/null +++ b/dev-tools/src/main/resources/forbidden/#all-signatures.txt# @@ -0,0 +1,129 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +@defaultMessage Convert to URI +java.net.URL#getPath() +java.net.URL#getFile() + +@defaultMessage Usage of getLocalHost is discouraged +java.net.InetAddress#getLocalHost() + +@defaultMessage Use java.nio.file instead of java.io.File API +java.util.jar.JarFile +java.util.zip.ZipFile +java.io.File +java.io.FileInputStream +java.io.FileOutputStream +java.io.PrintStream#(java.lang.String,java.lang.String) +java.io.PrintWriter#(java.lang.String,java.lang.String) +java.util.Formatter#(java.lang.String,java.lang.String,java.util.Locale) +java.io.RandomAccessFile +java.nio.file.Path#toFile() + +@defaultMessage Don't use deprecated lucene apis +org.apache.lucene.index.DocsEnum +org.apache.lucene.index.DocsAndPositionsEnum +org.apache.lucene.queries.TermFilter +org.apache.lucene.queries.TermsFilter +org.apache.lucene.search.TermRangeFilter +org.apache.lucene.search.NumericRangeFilter +org.apache.lucene.search.PrefixFilter + +java.nio.file.Paths @ Use PathUtils.get instead. +java.nio.file.FileSystems#getDefault() @ use PathUtils.getDefault instead. + +@defaultMessage Specify a location for the temp file/directory instead. +java.nio.file.Files#createTempDirectory(java.lang.String,java.nio.file.attribute.FileAttribute[]) +java.nio.file.Files#createTempFile(java.lang.String,java.lang.String,java.nio.file.attribute.FileAttribute[]) + +@defaultMessage Don't use java serialization - this can break BWC without noticing it +java.io.ObjectOutputStream +java.io.ObjectOutput +java.io.ObjectInputStream +java.io.ObjectInput + +java.nio.file.Files#isHidden(java.nio.file.Path) @ Dependent on the operating system, use FileSystemUtils.isHidden instead + +java.nio.file.Files#getFileStore(java.nio.file.Path) @ Use Environment.getFileStore() instead, impacted by JDK-8034057 +java.nio.file.Files#isWritable(java.nio.file.Path) @ Use Environment.isWritable() instead, impacted by JDK-8034057 + +@defaultMessage Resolve hosts explicitly to the address(es) you want with InetAddress. +java.net.InetSocketAddress#(java.lang.String,int) +java.net.Socket#(java.lang.String,int) +java.net.Socket#(java.lang.String,int,java.net.InetAddress,int) + +@defaultMessage Don't bind to wildcard addresses. Be specific. +java.net.DatagramSocket#() +java.net.DatagramSocket#(int) +java.net.InetSocketAddress#(int) +java.net.MulticastSocket#() +java.net.MulticastSocket#(int) +java.net.ServerSocket#(int) +java.net.ServerSocket#(int,int) + +@defaultMessage use NetworkAddress format/formatAddress to print IP or IP+ports +java.net.InetAddress#toString() +java.net.InetAddress#getHostAddress() +java.net.Inet4Address#getHostAddress() +java.net.Inet6Address#getHostAddress() +java.net.InetSocketAddress#toString() + +@defaultMessage avoid DNS lookups by accident: if you have a valid reason, then @SuppressWarnings with that reason so its completely clear +java.net.InetAddress#getHostName() +java.net.InetAddress#getCanonicalHostName() + +java.net.InetSocketAddress#getHostName() @ Use getHostString() instead, which avoids a DNS lookup + +@defaultMessage avoid adding additional dependencies on Guava +com.google.common.collect.Lists +com.google.common.collect.ImmutableList +com.google.common.base.Objects +com.google.common.base.Predicate +com.google.common.base.Predicates +com.google.common.base.Strings +com.google.common.base.Throwables +com.google.common.collect.Maps +com.google.common.collect.Sets +com.google.common.base.Preconditions +com.google.common.collect.ImmutableSortedSet +com.google.common.collect.Queues +com.google.common.util.concurrent.ListenableFuture +com.google.common.util.concurrent.SettableFuture +com.google.common.util.concurrent.Futures +com.google.common.util.concurrent.MoreExecutors +com.google.common.collect.ImmutableSortedMap +com.google.common.base.Charsets +com.google.common.base.Function +com.google.common.collect.Collections2 +com.google.common.cache.LoadingCache +com.google.common.cache.CacheLoader +com.google.common.collect.Iterables +<<<<<<< HEAD +com.google.common.util.concurrent.UncheckedExecutionException +com.google.common.util.concurrent.AtomicLongMap +com.google.common.primitives.Longs +com.google.common.io.ByteStreams +com.google.common.collect.UnmodifiableIterator +com.google.common.collect.ObjectArrays +com.google.common.collect.Multimap +com.google.common.collect.MultimapBuilder +======= +com.google.common.base.Joiner +>>>>>>> Remove and forbid use of com.google.common.base.Joiner + +@defaultMessage Do not violate java's access system +java.lang.reflect.AccessibleObject#setAccessible(boolean) +java.lang.reflect.AccessibleObject#setAccessible(java.lang.reflect.AccessibleObject[], boolean) diff --git a/dev-tools/src/main/resources/forbidden/all-signatures.txt b/dev-tools/src/main/resources/forbidden/all-signatures.txt index c133fd51004..8ab331b6b8c 100644 --- a/dev-tools/src/main/resources/forbidden/all-signatures.txt +++ b/dev-tools/src/main/resources/forbidden/all-signatures.txt @@ -120,6 +120,7 @@ com.google.common.collect.ObjectArrays com.google.common.collect.Multimap com.google.common.collect.MultimapBuilder com.google.common.math.LongMath +com.google.common.base.Joiner @defaultMessage Do not violate java's access system java.lang.reflect.AccessibleObject#setAccessible(boolean) From 2833ad4d6fc65a0104d6311394a01f27e0aac579 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 15 Sep 2015 10:37:57 -0400 Subject: [PATCH 16/33] Remove unused import in o.e.t.ESIntegTestCase --- core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java b/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java index 59fe63a06a5..592f65e31b9 100644 --- a/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java @@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.Randomness; import com.carrotsearch.randomizedtesting.annotations.TestGroup; import com.carrotsearch.randomizedtesting.generators.RandomInts; import com.carrotsearch.randomizedtesting.generators.RandomPicks; -import com.google.common.base.Joiner; import org.apache.http.impl.client.HttpClients; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; From b912ad7d56bec476aaa90781e1e671381ef77820 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 15 Sep 2015 10:41:15 -0400 Subject: [PATCH 17/33] Remove file that was inadvertently committed --- .../resources/forbidden/#all-signatures.txt# | 129 ------------------ 1 file changed, 129 deletions(-) delete mode 100644 dev-tools/src/main/resources/forbidden/#all-signatures.txt# diff --git a/dev-tools/src/main/resources/forbidden/#all-signatures.txt# b/dev-tools/src/main/resources/forbidden/#all-signatures.txt# deleted file mode 100644 index c3fa24045e2..00000000000 --- a/dev-tools/src/main/resources/forbidden/#all-signatures.txt# +++ /dev/null @@ -1,129 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -@defaultMessage Convert to URI -java.net.URL#getPath() -java.net.URL#getFile() - -@defaultMessage Usage of getLocalHost is discouraged -java.net.InetAddress#getLocalHost() - -@defaultMessage Use java.nio.file instead of java.io.File API -java.util.jar.JarFile -java.util.zip.ZipFile -java.io.File -java.io.FileInputStream -java.io.FileOutputStream -java.io.PrintStream#(java.lang.String,java.lang.String) -java.io.PrintWriter#(java.lang.String,java.lang.String) -java.util.Formatter#(java.lang.String,java.lang.String,java.util.Locale) -java.io.RandomAccessFile -java.nio.file.Path#toFile() - -@defaultMessage Don't use deprecated lucene apis -org.apache.lucene.index.DocsEnum -org.apache.lucene.index.DocsAndPositionsEnum -org.apache.lucene.queries.TermFilter -org.apache.lucene.queries.TermsFilter -org.apache.lucene.search.TermRangeFilter -org.apache.lucene.search.NumericRangeFilter -org.apache.lucene.search.PrefixFilter - -java.nio.file.Paths @ Use PathUtils.get instead. -java.nio.file.FileSystems#getDefault() @ use PathUtils.getDefault instead. - -@defaultMessage Specify a location for the temp file/directory instead. -java.nio.file.Files#createTempDirectory(java.lang.String,java.nio.file.attribute.FileAttribute[]) -java.nio.file.Files#createTempFile(java.lang.String,java.lang.String,java.nio.file.attribute.FileAttribute[]) - -@defaultMessage Don't use java serialization - this can break BWC without noticing it -java.io.ObjectOutputStream -java.io.ObjectOutput -java.io.ObjectInputStream -java.io.ObjectInput - -java.nio.file.Files#isHidden(java.nio.file.Path) @ Dependent on the operating system, use FileSystemUtils.isHidden instead - -java.nio.file.Files#getFileStore(java.nio.file.Path) @ Use Environment.getFileStore() instead, impacted by JDK-8034057 -java.nio.file.Files#isWritable(java.nio.file.Path) @ Use Environment.isWritable() instead, impacted by JDK-8034057 - -@defaultMessage Resolve hosts explicitly to the address(es) you want with InetAddress. -java.net.InetSocketAddress#(java.lang.String,int) -java.net.Socket#(java.lang.String,int) -java.net.Socket#(java.lang.String,int,java.net.InetAddress,int) - -@defaultMessage Don't bind to wildcard addresses. Be specific. -java.net.DatagramSocket#() -java.net.DatagramSocket#(int) -java.net.InetSocketAddress#(int) -java.net.MulticastSocket#() -java.net.MulticastSocket#(int) -java.net.ServerSocket#(int) -java.net.ServerSocket#(int,int) - -@defaultMessage use NetworkAddress format/formatAddress to print IP or IP+ports -java.net.InetAddress#toString() -java.net.InetAddress#getHostAddress() -java.net.Inet4Address#getHostAddress() -java.net.Inet6Address#getHostAddress() -java.net.InetSocketAddress#toString() - -@defaultMessage avoid DNS lookups by accident: if you have a valid reason, then @SuppressWarnings with that reason so its completely clear -java.net.InetAddress#getHostName() -java.net.InetAddress#getCanonicalHostName() - -java.net.InetSocketAddress#getHostName() @ Use getHostString() instead, which avoids a DNS lookup - -@defaultMessage avoid adding additional dependencies on Guava -com.google.common.collect.Lists -com.google.common.collect.ImmutableList -com.google.common.base.Objects -com.google.common.base.Predicate -com.google.common.base.Predicates -com.google.common.base.Strings -com.google.common.base.Throwables -com.google.common.collect.Maps -com.google.common.collect.Sets -com.google.common.base.Preconditions -com.google.common.collect.ImmutableSortedSet -com.google.common.collect.Queues -com.google.common.util.concurrent.ListenableFuture -com.google.common.util.concurrent.SettableFuture -com.google.common.util.concurrent.Futures -com.google.common.util.concurrent.MoreExecutors -com.google.common.collect.ImmutableSortedMap -com.google.common.base.Charsets -com.google.common.base.Function -com.google.common.collect.Collections2 -com.google.common.cache.LoadingCache -com.google.common.cache.CacheLoader -com.google.common.collect.Iterables -<<<<<<< HEAD -com.google.common.util.concurrent.UncheckedExecutionException -com.google.common.util.concurrent.AtomicLongMap -com.google.common.primitives.Longs -com.google.common.io.ByteStreams -com.google.common.collect.UnmodifiableIterator -com.google.common.collect.ObjectArrays -com.google.common.collect.Multimap -com.google.common.collect.MultimapBuilder -======= -com.google.common.base.Joiner ->>>>>>> Remove and forbid use of com.google.common.base.Joiner - -@defaultMessage Do not violate java's access system -java.lang.reflect.AccessibleObject#setAccessible(boolean) -java.lang.reflect.AccessibleObject#setAccessible(java.lang.reflect.AccessibleObject[], boolean) From 2357e65fcc7d9856cc3e59e77b7846cea4068e6b Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 15 Sep 2015 10:46:59 -0400 Subject: [PATCH 18/33] Remove unused import in o.e.a.t.TermVectorsFilter --- .../org/elasticsearch/action/termvectors/TermVectorsFilter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFilter.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFilter.java index f014fc82a16..a33e8e2cd42 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFilter.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFilter.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.action.termvectors; -import com.google.common.util.concurrent.AtomicLongMap; import org.apache.lucene.index.*; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.similarities.DefaultSimilarity; From 3d4fac6256c77aafdccc31c71d238e9d0187abef Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 15 Sep 2015 10:48:01 -0400 Subject: [PATCH 19/33] Remove unused import in o.e.t.UpdateThreadPoolSettingsTests --- .../elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java b/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java index 359ebddd1b7..562adaa8a0e 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.threadpool; -import com.google.common.util.concurrent.MoreExecutors; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; import org.elasticsearch.test.ESTestCase; From 1e209e3802a693414a18bfddbb917044c50e5a6a Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Tue, 15 Sep 2015 17:04:42 +0200 Subject: [PATCH 20/33] Plugins: Remove java opts/es java opts from plugin manager ... and run as client VM. Reasoning: When calling the plugin manager on java 7 with additional JAVA_OPTS that change heap configuration compared to what is set at the plugin manager shell script. This resulted in errors. This commit removes the JAVA_OPTS and ES_JAVA_OPTS from the plugin manager call to prevent those settings. Closes #12479 --- distribution/src/main/resources/bin/plugin | 2 +- distribution/src/main/resources/bin/plugin.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/src/main/resources/bin/plugin b/distribution/src/main/resources/bin/plugin index 973d2f6300d..d2d112c2f55 100755 --- a/distribution/src/main/resources/bin/plugin +++ b/distribution/src/main/resources/bin/plugin @@ -108,4 +108,4 @@ fi HOSTNAME=`hostname | cut -d. -f1` export HOSTNAME -eval "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Xmx64m -Xms16m -Delasticsearch -Des.path.home="\"$ES_HOME\"" $properties -cp "\"$ES_HOME/lib/*\"" org.elasticsearch.plugins.PluginManagerCliParser $args +eval "$JAVA" -client -Delasticsearch -Des.path.home="\"$ES_HOME\"" $properties -cp "\"$ES_HOME/lib/*\"" org.elasticsearch.plugins.PluginManagerCliParser $args diff --git a/distribution/src/main/resources/bin/plugin.bat b/distribution/src/main/resources/bin/plugin.bat index a5e593ceeeb..5984089fa34 100644 --- a/distribution/src/main/resources/bin/plugin.bat +++ b/distribution/src/main/resources/bin/plugin.bat @@ -11,7 +11,7 @@ TITLE Elasticsearch Plugin Manager ${project.version} SET HOSTNAME=%COMPUTERNAME% -"%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% -Xmx64m -Xms16m -Des.path.home="%ES_HOME%" -cp "%ES_HOME%/lib/*;" "org.elasticsearch.plugins.PluginManagerCliParser" %* +"%JAVA_HOME%\bin\java" -client -Des.path.home="%ES_HOME%" -cp "%ES_HOME%/lib/*;" "org.elasticsearch.plugins.PluginManagerCliParser" %* goto finally From dea0243193bcc6435a42fa038bd777116ef41591 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Mon, 14 Sep 2015 19:01:46 +0200 Subject: [PATCH 21/33] Release script: Major Improvements In order to be able to clean up broken steps in the release, one should be able to run each step individually. So now one needs to specifiy each step to run via cmdline argument * --deploy-sonatype: runs `mvn deploy` and pushes to the staging mvn repo * --deploy-s3: copies all artifacts over to the s3bucket * --deploy-s3-repos: Creates the s3 repositories * --no-install option, so that an existing mvn repo can be used for s3 operations Also, several minor changes have been added * Fixed typo in smoke test reference for email * Checking for correct AWS environment variables * Only create releases directory if it does not exist * RPM sign check is only executed after `mvn install` * Various path fixes for deb/rpm-s3 uploads * Added output of deb/rpm-s3 commands for easier debugging * Add configurable destination s3 bucket * Removed verbosity, always be verbose * Added color to the command which is being running right now, to differentiate from console output --- dev-tools/prepare_release_candidate.py | 208 ++++++++++++++----------- 1 file changed, 118 insertions(+), 90 deletions(-) diff --git a/dev-tools/prepare_release_candidate.py b/dev-tools/prepare_release_candidate.py index 51387cc6d89..5f8389372c4 100644 --- a/dev-tools/prepare_release_candidate.py +++ b/dev-tools/prepare_release_candidate.py @@ -63,7 +63,7 @@ To install the deb from an APT repo: APT line sources.list line: -deb http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/debian/ stable main +deb http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/debian/ stable main To install the RPM, create a YUM file like: @@ -73,30 +73,34 @@ containing: [elasticsearch-2.0] name=Elasticsearch repository for packages -baseurl=http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/centos +baseurl=http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/centos gpgcheck=1 gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch enabled=1 To smoke-test the release please run: - python3 -B ./dev-tools/smoke_tests_rc.py --version %(version)s --hash %(hash)s --plugins license,shield,watcher + python3 -B ./dev-tools/smoke_test_rc.py --version %(version)s --hash %(hash)s --plugins license,shield,watcher NOTE: this script requires JAVA_HOME to point to a Java 7 Runtime [1] https://github.com/elastic/elasticsearch/commit/%(hash)s -[2] http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/zip/elasticsearch/%(version)s/elasticsearch-%(version)s.zip -[3] http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/tar/elasticsearch/%(version)s/elasticsearch-%(version)s.tar.gz -[4] http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/rpm/elasticsearch/%(version)s/elasticsearch-%(version)s.rpm -[5] http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/deb/elasticsearch/%(version)s/elasticsearch-%(version)s.deb +[2] http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/zip/elasticsearch/%(version)s/elasticsearch-%(version)s.zip +[3] http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/tar/elasticsearch/%(version)s/elasticsearch-%(version)s.tar.gz +[4] http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/rpm/elasticsearch/%(version)s/elasticsearch-%(version)s.rpm +[5] http://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/elasticsearch/distribution/deb/elasticsearch/%(version)s/elasticsearch-%(version)s.deb """ -VERBOSE=True -def run(command, env_vars=None, verbose=VERBOSE): + +# console colors +COLOR_OK = '\033[92m' +COLOR_END = '\033[0m' +COLOR_FAIL = '\033[91m' + +def run(command, env_vars=None): if env_vars: for key, value in env_vars.items(): os.putenv(key, value) - if not verbose: - command = '%s >> /dev/null 2>&1' % (command) + print('*** Running: %s%s%s' % (COLOR_OK, command, COLOR_END)) if os.system(command): raise RuntimeError(' FAILED: %s' % (command)) @@ -192,10 +196,6 @@ def check_command_exists(name, cmd): except subprocess.CalledProcessError: raise RuntimeError('Could not run command %s - please make sure it is installed and in $PATH' % (name)) -# console colors -COLOR_OK = '\033[92m' -COLOR_END = '\033[0m' -COLOR_FAIL = '\033[91m' def run_and_print(text, run_function): try: print(text, end='') @@ -218,8 +218,8 @@ def check_env_var(text, env_var): def check_environment_and_commandline_tools(check_only): checks = list() - checks.append(check_env_var('Checking for AWS env configuration AWS_SECRET_ACCESS_KEY_ID... ', 'AWS_SECRET_ACCESS_KEY')) - checks.append(check_env_var('Checking for AWS env configuration AWS_ACCESS_KEY_ID... ', 'AWS_ACCESS_KEY_ID')) + checks.append(check_env_var('Checking for AWS env configuration AWS_SECRET_KEY... ', 'AWS_SECRET_KEY')) + checks.append(check_env_var('Checking for AWS env configuration AWS_ACCESS_KEY... ', 'AWS_ACCESS_KEY')) checks.append(run_and_print('Checking command: rpm... ', partial(check_command_exists, 'rpm', 'rpm --version'))) checks.append(run_and_print('Checking command: dpkg... ', partial(check_command_exists, 'dpkg', 'dpkg --version'))) checks.append(run_and_print('Checking command: gpg... ', partial(check_command_exists, 'gpg', 'gpg --version'))) @@ -238,37 +238,66 @@ def check_environment_and_commandline_tools(check_only): if __name__ == "__main__": parser = argparse.ArgumentParser(description='Builds and publishes a Elasticsearch Release') - parser.add_argument('--deploy', '-d', dest='deploy', action='store_true', - help='Installs and Deploys the release on a sonartype staging repository.') - parser.add_argument('--skipDocCheck', '-c', dest='skip_doc_check', action='store_false', - help='Skips any checks for pending documentation changes') - parser.add_argument('--push-s3', '-p', dest='push', action='store_true', + parser.add_argument('--deploy-sonatype', dest='deploy_sonatype', action='store_true', + help='Installs and Deploys the release on a sonatype staging repository.') + parser.add_argument('--deploy-s3', dest='deploy_s3', action='store_true', help='Pushes artifacts to the S3 staging area') - parser.add_argument('--install_only', '-i', dest='install_only', action='store_true', - help='Only runs a maven install to skip the remove deployment step') - parser.add_argument('--gpg-key', '-k', dest='gpg_key', default="D88E42B4", + parser.add_argument('--deploy-s3-repos', dest='deploy_s3_repos', action='store_true', + help='Creates package repositories in S3 repo') + parser.add_argument('--no-install', dest='no_install', action='store_true', + help='Does not run "mvn install", expects this to be run already and reuses artifacts from local repo, only useful with --deploy-s3/--deploy-s3-repos, after sonatype deplomeny to ensure same artifacts') + parser.add_argument('--skip-doc-check', dest='skip_doc_check', action='store_false', + help='Skips any checks for pending documentation changes') + parser.add_argument('--skip-tests', dest='skip_tests', action='store_true', + help='Skips any test runs') + parser.add_argument('--gpg-key', dest='gpg_key', default="D88E42B4", help='Allows you to specify a different gpg_key to be used instead of the default release key') - parser.add_argument('--verbose', '-b', dest='verbose', - help='Runs the script in verbose mode') - parser.add_argument('--check-only', dest='check_only', action='store_true', + parser.add_argument('--bucket', '-b', dest='bucket', default="download.elasticsearch.org", + help='Allows you to specify a different s3 bucket to upload the artifacts to') + parser.add_argument('--quiet', dest='quiet', action='store_true', + help='Runs the script in quiet mode') + parser.add_argument('--check', dest='check', action='store_true', help='Checks and reports for all requirements and then exits') - parser.set_defaults(deploy=False) - parser.set_defaults(skip_doc_check=False) - parser.set_defaults(push=False) - parser.set_defaults(install_only=False) - parser.set_defaults(verbose=False) - args = parser.parse_args() - install_and_deploy = args.deploy - skip_doc_check = args.skip_doc_check - push = args.push - gpg_key = args.gpg_key - install_only = args.install_only - VERBOSE = args.verbose - check_environment_and_commandline_tools(args.check_only) + # by default, we only run mvn install and dont push anything repo + parser.set_defaults(deploy_sonatype=False) + parser.set_defaults(deploy_s3=False) + parser.set_defaults(deploy_s3_repos=False) + parser.set_defaults(no_install=False) + # other defaults + parser.set_defaults(skip_doc_check=False) + parser.set_defaults(quiet=False) + parser.set_defaults(skip_tests=False) + + args = parser.parse_args() + skip_doc_check = args.skip_doc_check + gpg_key = args.gpg_key + bucket = args.bucket + deploy_sonatype = args.deploy_sonatype + deploy_s3 = args.deploy_s3 + deploy_s3_repos = args.deploy_s3_repos + run_mvn_install = not args.no_install + skip_tests = args.skip_tests + + check_environment_and_commandline_tools(args.check) + + if not run_mvn_install and deploy_sonatype: + print('Using --no-install and --deploy-sonatype together does not work. Exiting') + sys.exit(-1) + + print('*** Preparing a release candidate: ', end='') + print('deploy sonatype: %s%s%s' % (COLOR_OK if deploy_sonatype else COLOR_FAIL, 'yes' if deploy_sonatype else 'no', COLOR_END), end='') + print(', deploy s3: %s%s%s' % (COLOR_OK if deploy_s3 else COLOR_FAIL, 'yes' if deploy_s3 else 'no', COLOR_END), end='') + print(', deploy s3 repos: %s%s%s' % (COLOR_OK if deploy_s3_repos else COLOR_FAIL, 'yes' if deploy_s3_repos else 'no', COLOR_END), end='') + print('') + + shortHash = subprocess.check_output('git log --pretty=format:"%h" -n 1', shell=True).decode('utf-8') + releaseDirectory = os.getenv('HOME') + '/elastic-releases' + release_version = find_release_version() + localRepo = '%s/elasticsearch-%s-%s' % (releaseDirectory, release_version, shortHash) + localRepoElasticsearch = localRepo + '/org/elasticsearch' ensure_checkout_is_clean() - release_version = find_release_version() if not re.match('(\d+\.\d+)\.*',release_version): raise RuntimeError('illegal release version format: %s' % (release_version)) major_minor_version = re.match('(\d+\.\d+)\.*',release_version).group(1) @@ -281,7 +310,6 @@ if __name__ == "__main__": if pending_files: raise RuntimeError('pending coming[%s] documentation changes found in %s' % (release_version, pending_files)) - run('cd dev-tools && mvn versions:set -DnewVersion=%s -DgenerateBackupPoms=false' % (release_version)) run('cd rest-api-spec && mvn versions:set -DnewVersion=%s -DgenerateBackupPoms=false' % (release_version)) run('mvn versions:set -DnewVersion=%s -DgenerateBackupPoms=false' % (release_version)) @@ -290,70 +318,69 @@ if __name__ == "__main__": print('*** Done removing snapshot version. DO NOT COMMIT THIS, WHEN CREATING A RELEASE CANDIDATE.') - shortHash = subprocess.check_output('git log --pretty=format:"%h" -n 1', shell=True).decode('utf-8') - releaseDirectory = os.getenv('HOME') + '/elastic-releases' - os.mkdir(releaseDirectory) - localRepo = '%s/elasticsearch-%s-%s' % (releaseDirectory, release_version, shortHash) - localRepoElasticsearch = localRepo + '/org/elasticsearch' - if os.path.exists(localRepoElasticsearch): + if not os.path.exists(releaseDirectory): + os.mkdir(releaseDirectory) + if os.path.exists(localRepoElasticsearch) and run_mvn_install: print('clean local repository %s' % localRepoElasticsearch) shutil.rmtree(localRepoElasticsearch) - if install_only: - mvn_target = 'install' - else: - mvn_target = 'deploy' - install_command = 'mvn clean %s -Prelease -Dskip.integ.tests=true -Dgpg.key="%s" -Dpackaging.rpm.rpmbuild=/usr/bin/rpmbuild -Drpm.sign=true -Dmaven.repo.local=%s -Dno.commit.pattern="\\bno(n|)commit\\b" -Dforbidden.test.signatures=""' % (mvn_target, gpg_key, localRepo) + mvn_target = 'deploy' if deploy_sonatype else 'install' + tests = '-DskipTests' if skip_tests else '-Dskip.integ.tests=true' + install_command = 'mvn clean %s -Prelease %s -Dgpg.key="%s" -Dpackaging.rpm.rpmbuild=/usr/bin/rpmbuild -Drpm.sign=true -Dmaven.repo.local=%s -Dno.commit.pattern="\\bno(n|)commit\\b" -Dforbidden.test.signatures=""' % (mvn_target, tests, gpg_key, localRepo) clean_repo_command = 'find %s -name _remote.repositories -exec rm {} \;' % (localRepoElasticsearch) - rename_metadata_files_command = 'for i in $(find %s -name "maven-metadata-local.xml*") ; do mv "$i" "${i/-local/}" ; done' % (localRepoElasticsearch) - if install_and_deploy: + + if not run_mvn_install: + print('') + print('*** By choosing --no-install we assume you ran the following commands successfully:') + print(' %s' % (install_command)) + print(' 1. Remove all _remote.repositories: %s' % (clean_repo_command)) + rename_metadata_files_command = 'for i in $(find %s -name "maven-metadata-local.xml*") ; do mv "$i" "${i/-local/}" ; done' % (localRepoElasticsearch) + print(' 2. Rename all maven metadata files: %s' % (rename_metadata_files_command)) + else: for cmd in [install_command, clean_repo_command]: run(cmd) rename_local_meta_files(localRepoElasticsearch) - else: - print('') - print('*** To create a release candidate run: ') - print(' %s' % (install_command)) - print(' 1. Remove all _remote.repositories: %s' % (clean_repo_command)) - print(' 2. Rename all maven metadata files: %s' % (rename_metadata_files_command)) - print('Ensuring that RPM has been signed') rpm = '%s/distribution/rpm/elasticsearch/%s/elasticsearch-%s.rpm' % (localRepoElasticsearch, release_version, release_version) + print('Ensuring that RPM has been signed') ensure_rpm_is_signed(rpm, gpg_key) # repository push commands - s3cmd_sync_to_staging_bucket_cmd = 's3cmd sync -P %s s3://download.elasticsearch.org/elasticsearch/staging/%s-%s/org/' % (localRepoElasticsearch, release_version, shortHash) - s3_bucket_sync_to = 'download.elasticsearch.org/elasticsearch/staging/%s-%s/repos' % (release_version, shortHash) + s3cmd_sync_to_staging_bucket_cmd = 's3cmd sync -P %s s3://%s/elasticsearch/staging/%s-%s/org/' % (localRepoElasticsearch, bucket, release_version, shortHash) + s3_bucket_sync_to = '%s/elasticsearch/staging/%s-%s/repos/' % (bucket, release_version, shortHash) s3cmd_sync_official_repo_cmd = 's3cmd sync s3://packages.elasticsearch.org/elasticsearch/%s s3://%s' % (major_minor_version, s3_bucket_sync_to) - debs3_prefix = 'elasticsearch/staging/%s-%s/repos/debian' % (release_version, shortHash) - debs3_upload_cmd = 'deb-s3 upload --preserve-versions %s/distribution/deb/elasticsearch/%s/elasticsearch-%s.deb -b download.elasticsearch.org --prefix %s --sign %s --arch amd64' % (localRepoElasticsearch, release_version, release_version, prefix, gpg_key) - debs3_list_cmd = 'deb-s3 list -b download.elasticsearch.org --prefix %s' % (debs3_prefix) - debs3_verify_cmd = 'deb-s3 verify -b download.elasticsearch.org --prefix %s' % (debs3_prefix) - rpms3_prefix = 'elasticsearch/staging/%s-%s/repos/centos' % (release_version, shortHash) - rpms3_upload_cmd = 'rpm-s3 -v -b download.elasticsearch.org -p %s --sign --visibility public-read -k 0 %s' % (rpms3_prefix, rpm) + debs3_prefix = 'elasticsearch/staging/%s-%s/repos/%s/debian' % (release_version, shortHash, major_minor_version) + debs3_upload_cmd = 'deb-s3 upload --preserve-versions %s/distribution/deb/elasticsearch/%s/elasticsearch-%s.deb -b %s --prefix %s --sign %s --arch amd64' % (localRepoElasticsearch, release_version, release_version, bucket, debs3_prefix, gpg_key) + debs3_list_cmd = 'deb-s3 list -b %s --prefix %s' % (bucket, debs3_prefix) + debs3_verify_cmd = 'deb-s3 verify -b %s --prefix %s' % (bucket, debs3_prefix) + rpms3_prefix = 'elasticsearch/staging/%s-%s/repos/%s/centos' % (release_version, shortHash, major_minor_version) + rpms3_upload_cmd = 'rpm-s3 -v -b %s -p %s --sign --visibility public-read -k 0 %s' % (bucket, rpms3_prefix, rpm) - if push: + if deploy_s3: run(s3cmd_sync_to_staging_bucket_cmd) - print('Syncing official package repository into staging s3 bucket') - run(s3cmd_sync_official_repo_cmd) - print('Uploading debian package (you will be prompted for the passphrase!)') - run(debs3_upload_cmd) - run(debs3_list_cmd) - run(debs3_verify_cmd) - print('Uploading rpm package (you will be prompted for the passphrase!)') - run(rpms3_upload_cmd) else: print('') print('*** To push a release candidate to s3 run: ') print(' 1. Sync %s into S3 bucket' % (localRepoElasticsearch)) print (' %s' % (s3cmd_sync_to_staging_bucket_cmd)) - print(' 2. Create repositories: ') - print(' 1. Sync existing repo into staging: %s' % s3cmd_sync_official_repo_cmd) - print(' 2. Upload debian package (and sign it) %s' % debs3_upload_cmd) - print(' 3. List all debian packages: %s' % debs3_list_cmd) - print(' 4. Verify debian packages: %s' % debs3_verify_cmd) - print(' 5. Upload RPM: %s' % rpms3_upload_cmd) + + if deploy_s3_repos: + print('*** Syncing official package repository into staging s3 bucket') + run(s3cmd_sync_official_repo_cmd) + print('*** Uploading debian package (you will be prompted for the passphrase!)') + run(debs3_upload_cmd) + run(debs3_list_cmd) + run(debs3_verify_cmd) + print('*** Uploading rpm package (you will be prompted for the passphrase!)') + run(rpms3_upload_cmd) + else: + print('*** To create repositories on S3 run:') + print(' 1. Sync existing repo into staging: %s' % s3cmd_sync_official_repo_cmd) + print(' 2. Upload debian package (and sign it): %s' % debs3_upload_cmd) + print(' 3. List all debian packages: %s' % debs3_list_cmd) + print(' 4. Verify debian packages: %s' % debs3_verify_cmd) + print(' 5. Upload RPM: %s' % rpms3_upload_cmd) print('') print('NOTE: the above mvn command will promt you several times for the GPG passphrase of the key you specified you can alternatively pass it via -Dgpg.passphrase=yourPassPhrase') print(' since RPM signing doesn\'t support gpg-agents the recommended way to set the password is to add a release profile to your settings.xml:') @@ -368,18 +395,19 @@ if __name__ == "__main__": """) print('NOTE: Running s3cmd might require you to create a config file with your credentials, if the s3cmd does not support suppliying them via the command line!') + print('*** Once the release is deployed and published send out the following mail to dev@elastic.co:') - string_format_dict = {'version' : release_version, 'hash': shortHash, 'major_minor_version' : major_minor_version} + string_format_dict = {'version' : release_version, 'hash': shortHash, 'major_minor_version' : major_minor_version, 'bucket': bucket} print(MAIL_TEMPLATE % string_format_dict) print('') print('You can verify that pushing to the staging repository pushed all the artifacts by running (log into sonatype to find out the correct id):') - print(' python3 -B dev-tools/validate-maven-repository.py %s https://oss.sonatype.org/service/local/repositories/orgelasticsearch-IDTOFINDOUT/content/org/elasticsearch ' %(localElasticsearchRepo)) + print(' python -B dev-tools/validate-maven-repository.py %s https://oss.sonatype.org/service/local/repositories/orgelasticsearch-IDTOFINDOUT/content/org/elasticsearch ' %(localRepoElasticsearch)) print('') print('To publish the release and the repo on S3 execute the following commands:') - print(' s3cmd cp --recursive s3://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/ s3://packages.elasticsearch.org/elasticsearch/%(major_minor_version)s' % string_format_dict) - print(' s3cmd cp --recursive s3://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/org/ s3://download.elasticsearch.org/elasticsearch/release/org' % string_format_dict) + print(' s3cmd cp --recursive s3://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/ s3://packages.elasticsearch.org/elasticsearch/%(major_minor_version)s' % string_format_dict) + print(' s3cmd cp --recursive s3://%(bucket)s/elasticsearch/staging/%(version)s-%(hash)s/org/ s3://%(bucket)s/elasticsearch/release/org' % string_format_dict) print('Now go ahead and tag the release:') print(' git tag -a v%(version)s %(hash)s' % string_format_dict) print(' git push origin v%(version)s' % string_format_dict ) From 257f3e862f5cd0cd24777efe51bfae6d1d03fedf Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 15 Sep 2015 11:15:21 -0400 Subject: [PATCH 22/33] Add missing forbidden entries for removed Guava classes --- dev-tools/src/main/resources/forbidden/all-signatures.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev-tools/src/main/resources/forbidden/all-signatures.txt b/dev-tools/src/main/resources/forbidden/all-signatures.txt index 8ab331b6b8c..613ed07112d 100644 --- a/dev-tools/src/main/resources/forbidden/all-signatures.txt +++ b/dev-tools/src/main/resources/forbidden/all-signatures.txt @@ -121,6 +121,10 @@ com.google.common.collect.Multimap com.google.common.collect.MultimapBuilder com.google.common.math.LongMath com.google.common.base.Joiner +com.google.common.collect.ArrayListMultimap +com.google.common.collect.HashMultimap +com.google.common.collect.FluentIterable +com.google.common.io.Files @defaultMessage Do not violate java's access system java.lang.reflect.AccessibleObject#setAccessible(boolean) From 589fb9674f67b0f2bd8f42199189b22e02d0de7c Mon Sep 17 00:00:00 2001 From: Nicholas Knize Date: Tue, 15 Sep 2015 12:35:10 -0500 Subject: [PATCH 23/33] Relax geopoint comparison tolerance Weighted centroid, morton hash, and geohash can be imprecise (computation error) to 1e-5. The previous compareTo set this tolerance too strict (1e-6) causing a reproducible comparison error on weighted centroid (#13558). This change relaxes the tolerance to the acceptable computation error of 1e-5 closes #13558 --- core/src/main/java/org/apache/lucene/util/XGeoUtils.java | 2 +- .../elasticsearch/search/aggregations/bucket/GeoHashGridIT.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/lucene/util/XGeoUtils.java b/core/src/main/java/org/apache/lucene/util/XGeoUtils.java index 7e97306cc41..df22e377949 100644 --- a/core/src/main/java/org/apache/lucene/util/XGeoUtils.java +++ b/core/src/main/java/org/apache/lucene/util/XGeoUtils.java @@ -30,7 +30,7 @@ public final class XGeoUtils { public static final short BITS = 31; private static final double LON_SCALE = (0x1L< Date: Mon, 7 Sep 2015 18:48:22 -0700 Subject: [PATCH 24/33] IndexNameExpressionResolver should not ignore any wildcards that occur elsewhere in the expression, closes #13334 --- .../metadata/IndexNameExpressionResolver.java | 2 +- .../WildcardExpressionResolverTests.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index b90babef876..5c7bb8e2b80 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -587,7 +587,7 @@ public class IndexNameExpressionResolver extends AbstractComponent { if (Regex.isMatchAllPattern(expression)) { // Can only happen if the expressions was initially: '-*' matches = metaData.getAliasAndIndexLookup(); - } else if (expression.endsWith("*")) { + } else if (expression.indexOf("*") == expression.length() - 1) { // Suffix wildcard: assert expression.length() >= 2 : "expression [" + expression + "] should have at least a length of 2"; String fromPrefix = expression.substring(0, expression.length() - 1); diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java index 4c364151abb..fa6217e3d1f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java @@ -90,6 +90,28 @@ public class WildcardExpressionResolverTests extends ESTestCase { assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testX*"))), equalTo(newHashSet("testXXX", "testXXY"))); } + // issue #13334 + public void testMultipleWildcards() { + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder("testXXX")) + .put(indexBuilder("testXXY")) + .put(indexBuilder("testXYY")) + .put(indexBuilder("testYYY")) + .put(indexBuilder("kuku")) + .put(indexBuilder("kukuYYY")); + + ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build(); + IndexNameExpressionResolver.WildcardExpressionResolver resolver = new IndexNameExpressionResolver.WildcardExpressionResolver(); + + IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen()); + assertThat(newHashSet(resolver.resolve(context, Arrays.asList("test*X*"))), equalTo(newHashSet("testXXX", "testXXY", "testXYY"))); + assertThat(newHashSet(resolver.resolve(context, Arrays.asList("test*X*Y"))), equalTo(newHashSet("testXXY", "testXYY"))); + assertThat(newHashSet(resolver.resolve(context, Arrays.asList("kuku*Y*"))), equalTo(newHashSet("kukuYYY"))); + assertThat(newHashSet(resolver.resolve(context, Arrays.asList("*Y*"))), equalTo(newHashSet("testXXY", "testXYY", "testYYY", "kukuYYY"))); + assertThat(newHashSet(resolver.resolve(context, Arrays.asList("test*Y*X"))).size(), equalTo(0)); + assertThat(newHashSet(resolver.resolve(context, Arrays.asList("*Y*X"))).size(), equalTo(0)); + } + private IndexMetaData.Builder indexBuilder(String index) { return IndexMetaData.builder(index).settings(settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); } From bfa984e1c28b1b6840fbf46ad1d0d93433f14a78 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 15 Sep 2015 21:03:57 +0200 Subject: [PATCH 25/33] Prevent losing stacktraces when exceptions occur This commit removes unnecesssary use of ExceptionHelpers where we actually should serialize / deserialize the actual exception. This commit also fixes one of the oddest problems where the actual exception was never rendered / printed if `all shards failed` due to a missing cause. This commit unfortunately doesn't fix Snapshot/Restore which is almost unfixable since it has to serialize XContent and read from it which can't transport exceptions. --- .../elasticsearch/ElasticsearchException.java | 2 +- .../TransportMultiPercolateAction.java | 2 +- .../TransportShardMultiPercolateAction.java | 12 +- .../search/SearchPhaseExecutionException.java | 18 +++ .../termvectors/MultiTermVectorsResponse.java | 20 ++-- .../TransportMultiTermVectorsAction.java | 8 +- .../TransportShardMultiTermsVectorAction.java | 2 +- .../org/elasticsearch/index/store/Store.java | 53 +++++--- .../repositories/VerificationFailure.java | 10 +- .../VerifyNodeRepositoryAction.java | 4 +- .../GroovyScriptCompilationException.java | 4 +- .../groovy/GroovyScriptEngineService.java | 4 +- .../GroovyScriptExecutionException.java | 5 +- .../termvectors/MultiTermVectorsIT.java | 15 ++- .../elasticsearch/index/store/StoreTests.java | 113 ++++++++++++++++++ .../percolator/MultiPercolatorIT.java | 2 +- .../rest/BytesRestResponseTests.java | 3 + .../routing/SimpleRoutingIT.java | 4 +- .../script/GroovySecurityIT.java | 5 +- .../cloud/aws/network/Ec2NameResolver.java | 5 +- .../aws/node/Ec2CustomNodeAttributes.java | 3 +- 21 files changed, 227 insertions(+), 67 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java index 81521c601f3..857f7d7be32 100644 --- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -291,7 +291,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte /** * Renders a cause exception as xcontent */ - protected final void causeToXContent(XContentBuilder builder, Params params) throws IOException { + protected void causeToXContent(XContentBuilder builder, Params params) throws IOException { final Throwable cause = getCause(); if (cause != null && params.paramAsBoolean(REST_EXCEPTION_SKIP_CAUSE, REST_EXCEPTION_SKIP_CAUSE_DEFAULT) == false) { builder.field("caused_by"); diff --git a/core/src/main/java/org/elasticsearch/action/percolate/TransportMultiPercolateAction.java b/core/src/main/java/org/elasticsearch/action/percolate/TransportMultiPercolateAction.java index 3d8d5518be7..743b4dfe27b 100644 --- a/core/src/main/java/org/elasticsearch/action/percolate/TransportMultiPercolateAction.java +++ b/core/src/main/java/org/elasticsearch/action/percolate/TransportMultiPercolateAction.java @@ -251,7 +251,7 @@ public class TransportMultiPercolateAction extends HandledTransportAction Date: Tue, 15 Sep 2015 19:34:48 -0400 Subject: [PATCH 26/33] Tests: Make sure that the cluster state came from the same master before checking for discrepancies If nodes don't agree on the master, the cluster state doesn't have to match even if they have the same version --- .../test/java/org/elasticsearch/test/ESIntegTestCase.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java b/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java index 592f65e31b9..f561dd6df64 100644 --- a/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1099,8 +1099,9 @@ public abstract class ESIntegTestCase extends ESTestCase { localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null); final Map localStateMap = convertToMap(localClusterState); final int localClusterStateSize = localClusterState.toString().length(); - // Check that the non-master node has the same version of the cluster state as the master and that this node didn't disconnect from the master - if (masterClusterState.version() == localClusterState.version() && localClusterState.nodes().nodes().containsKey(masterId)) { + // Check that the non-master node has the same version of the cluster state as the master and + // that the master node matches the master (otherwise there is no requirement for the cluster state to match) + if (masterClusterState.version() == localClusterState.version() && masterId.equals(localClusterState.nodes().masterNodeId())) { try { assertEquals("clusterstate UUID does not match", masterClusterState.stateUUID(), localClusterState.stateUUID()); // We cannot compare serialization bytes since serialization order of maps is not guaranteed From 8d542da89dc179cd18f97a115ee36b17117a7abc Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 15 Sep 2015 22:54:29 -0400 Subject: [PATCH 27/33] Test: await fix on GroovyScriptIT.testGroovyExceptionSerialization --- core/src/test/java/org/elasticsearch/script/GroovyScriptIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/elasticsearch/script/GroovyScriptIT.java b/core/src/test/java/org/elasticsearch/script/GroovyScriptIT.java index c24d4999af7..58c1b3379d5 100644 --- a/core/src/test/java/org/elasticsearch/script/GroovyScriptIT.java +++ b/core/src/test/java/org/elasticsearch/script/GroovyScriptIT.java @@ -61,6 +61,7 @@ public class GroovyScriptIT extends ESIntegTestCase { } @Test + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/13599") public void testGroovyExceptionSerialization() throws Exception { List reqs = new ArrayList<>(); for (int i = 0; i < randomIntBetween(50, 500); i++) { From 76b027a97997dcc51dfbe16edf49bd0d40e9eb0c Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 16 Sep 2015 09:21:09 +0200 Subject: [PATCH 28/33] [TEST] Only assert on a generic AssertionError The specific error might be lost on the transport layer and turned into a generic AssertionError instead. Closes #13599 --- .../src/test/java/org/elasticsearch/script/GroovyScriptIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/script/GroovyScriptIT.java b/core/src/test/java/org/elasticsearch/script/GroovyScriptIT.java index 58c1b3379d5..6a35012c818 100644 --- a/core/src/test/java/org/elasticsearch/script/GroovyScriptIT.java +++ b/core/src/test/java/org/elasticsearch/script/GroovyScriptIT.java @@ -61,7 +61,6 @@ public class GroovyScriptIT extends ESIntegTestCase { } @Test - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/13599") public void testGroovyExceptionSerialization() throws Exception { List reqs = new ArrayList<>(); for (int i = 0; i < randomIntBetween(50, 500); i++) { @@ -93,7 +92,7 @@ public class GroovyScriptIT extends ESIntegTestCase { assertThat(e.toString() + "should have contained GroovyScriptExecutionException", e.toString().contains("GroovyScriptExecutionException"), equalTo(true)); assertThat(e.toString()+ "should have contained an assert error", - e.toString().contains("PowerAssertionError[assert false"), equalTo(true)); + e.toString().contains("AssertionError[assert false"), equalTo(true)); } } From 01e6d8e3dc23c0969d5337af27a3326cf74a55e0 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 16 Sep 2015 04:04:51 -0400 Subject: [PATCH 29/33] Remove java.lang.reflect.ReflectPermission "suppressAccessChecks" Closes #13603 Squashed commit of the following: commit 8799fb42d80297a79285beaf407b1bbecdb5854d Author: Robert Muir Date: Wed Sep 16 03:32:29 2015 -0400 Add randomizedtesting snapshot note commit 0d874d9f0f5fddaeab8f48f9816a052dcaa691be Author: Robert Muir Date: Wed Sep 16 03:11:01 2015 -0400 Add a mechanism for insecure plugins and get all tests passing commit 80540aeb9a264f6f299aaa3bc89df7f9b7923a60 Author: Robert Muir Date: Tue Sep 15 22:59:29 2015 -0400 Really remove, we are killing this commit 884818c1ad44ca2e7572a6998c086580be919657 Author: Robert Muir Date: Tue Sep 15 22:57:22 2015 -0400 fill in TODOs commit 34f4cb81f249edfec4d8d211da892f8c987e5948 Author: Robert Muir Date: Tue Sep 15 22:31:43 2015 -0400 Publish snapshots of RR and lucene and cutover commit d68eb9d66ce059761805c64d67e41a29098c9afa Merge: f27e208 f62da59 Author: Robert Muir Date: Tue Sep 15 12:32:41 2015 -0400 Merge branch 'master' into kill-setaccessible commit f27e20855216dab6a6ad035d41018d8c67f3144c Author: Robert Muir Date: Tue Sep 15 12:32:21 2015 -0400 make a real lucene snapshot --- .../bootstrap/BootstrapInfo.java | 13 +++++ .../org/elasticsearch/bootstrap/Security.java | 49 +++++++++++++++++++ .../elasticsearch/bootstrap/security.policy | 40 +++++++++++++-- .../bootstrap/BootstrapForTesting.java | 16 ++++++ ...ers-common-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ers-common-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...ard-codecs-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ard-codecs-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...ucene-core-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ucene-core-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...xpressions-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...xpressions-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...e-grouping-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...e-grouping-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...ighlighter-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ighlighter-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...ucene-join-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ucene-join-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...ene-memory-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ene-memory-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...ucene-misc-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ucene-misc-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...ne-queries-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ne-queries-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...ueryparser-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ueryparser-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...ne-sandbox-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ne-sandbox-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...ne-spatial-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ne-spatial-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...-spatial3d-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...-spatial3d-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...ne-suggest-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...ne-suggest-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...lyzers-icu-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...lyzers-icu-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...s-kuromoji-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...s-kuromoji-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...s-phonetic-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...s-phonetic-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...rs-smartcn-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...rs-smartcn-5.4.0-snapshot-1702855.jar.sha1 | 1 + ...rs-stempel-5.4.0-snapshot-1702265.jar.sha1 | 1 - ...rs-stempel-5.4.0-snapshot-1702855.jar.sha1 | 1 + .../cloud/gce/GceComputeServiceImpl.java | 13 ++++- .../discovery/ec2/Ec2DiscoveryPlugin.java | 19 +++++++ .../repository/s3/S3RepositoryPlugin.java | 19 +++++++ pom.xml | 23 ++++++++- 48 files changed, 206 insertions(+), 26 deletions(-) delete mode 100644 distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-core-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-core-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-expressions-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-expressions-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-grouping-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-grouping-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-highlighter-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-highlighter-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-join-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-join-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-memory-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-memory-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-misc-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-misc-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-queries-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-queries-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-queryparser-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-queryparser-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-sandbox-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-sandbox-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-spatial-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-spatial-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 distribution/licenses/lucene-suggest-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 distribution/licenses/lucene-suggest-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1702855.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1702265.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1702855.jar.sha1 diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java index 829c45f074e..abaa784d0b9 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java @@ -19,6 +19,9 @@ package org.elasticsearch.bootstrap; +import java.util.Collections; +import java.util.Set; + /** * Exposes system startup information */ @@ -43,4 +46,14 @@ public final class BootstrapInfo { public static boolean isMemoryLocked() { return Natives.isMemoryLocked(); } + + /** + * Returns set of insecure plugins. + *

+ * These are plugins with unresolved issues in third-party libraries, + * that require additional privileges as a workaround. + */ + public static Set getInsecurePluginList() { + return Collections.unmodifiableSet(Security.INSECURE_PLUGINS.keySet()); + } } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java index a6bcb2f3732..a8ed511be13 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java @@ -20,11 +20,14 @@ package org.elasticsearch.bootstrap; import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.env.Environment; import java.io.*; import java.net.URL; import java.nio.file.AccessMode; +import java.nio.file.DirectoryStream; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.NotDirectoryException; @@ -32,6 +35,7 @@ import java.nio.file.Path; import java.security.Permissions; import java.security.Policy; import java.util.Collections; +import java.util.HashMap; import java.util.IdentityHashMap; import java.util.Map; import java.util.regex.Pattern; @@ -95,6 +99,8 @@ final class Security { static void configure(Environment environment) throws Exception { // set properties for jar locations setCodebaseProperties(); + // set properties for problematic plugins + setPluginCodebaseProperties(environment); // enable security policy: union of template and environment-based paths. Policy.setPolicy(new ESPolicy(createPermissions(environment))); @@ -121,6 +127,9 @@ final class Security { static { Map m = new IdentityHashMap<>(); m.put(Pattern.compile(".*lucene-core-.*\\.jar$"), "es.security.jar.lucene.core"); + m.put(Pattern.compile(".*lucene-test-framework-.*\\.jar$"), "es.security.jar.lucene.testframework"); + m.put(Pattern.compile(".*randomizedtesting-runner-.*\\.jar$"), "es.security.jar.randomizedtesting.runner"); + m.put(Pattern.compile(".*junit4-ant-.*\\.jar$"), "es.security.jar.randomizedtesting.junit4"); m.put(Pattern.compile(".*securemock-.*\\.jar$"), "es.security.jar.elasticsearch.securemock"); SPECIAL_JARS = Collections.unmodifiableMap(m); } @@ -150,6 +159,46 @@ final class Security { } } + // mapping of insecure plugins to codebase properties + // note that this is only read once, when policy is parsed. + static final Map INSECURE_PLUGINS; + static { + Map m = new HashMap<>(); + m.put("repository-s3", "es.security.insecure.plugin.repository-s3"); + m.put("discovery-ec2", "es.security.insecure.plugin.discovery-ec2"); + m.put("cloud-gce", "es.security.insecure.plugin.cloud-gce" ); + INSECURE_PLUGINS = Collections.unmodifiableMap(m); + } + + /** + * Sets properties (codebase URLs) for policy files. + * we look for matching plugins and set URLs to fit + */ + @SuppressForbidden(reason = "proper use of URL") + static void setPluginCodebaseProperties(Environment environment) throws IOException { + if (Files.exists(environment.pluginsFile())) { + try (DirectoryStream stream = Files.newDirectoryStream(environment.pluginsFile())) { + for (Path plugin : stream) { + String prop = INSECURE_PLUGINS.get(plugin.getFileName().toString()); + if (prop != null) { + if (System.getProperty(prop) != null) { + throw new IllegalStateException("property: " + prop + " is unexpectedly set: " + System.getProperty(prop)); + } + System.setProperty(prop, plugin.toUri().toURL().toString() + "*"); + ESLogger logger = Loggers.getLogger(Security.class); + logger.warn("Adding permissions for insecure plugin [{}]", plugin.getFileName()); + logger.warn("There are unresolved issues with third-party code that may reduce the security of the system"); + } + } + } + } + for (String prop : INSECURE_PLUGINS.values()) { + if (System.getProperty(prop) == null) { + System.setProperty(prop, "file:/dev/null"); // no chance to be interpreted as "all" + } + } + } + /** returns dynamic Permissions to configured paths */ static Permissions createPermissions(Environment environment) throws IOException { Permissions policy = new Permissions(); diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index befeef41bbd..70346acad49 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -34,6 +34,27 @@ grant codeBase "file:${{java.ext.dirs}}/*" { grant codeBase "${es.security.jar.lucene.core}" { // needed to allow MMapDirectory's "unmap hack" permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; + +//// Insecure plugin permissions: +//// These are dangerous permissions due to problems in third-party library code +//// We try our best to contain and protect the issues, but ultimately warn the user +//// when installing these plugins that it can introduce a security risk + +grant codeBase "${es.security.insecure.plugin.repository-s3}" { + // needed because of problems in aws-sdk + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; + +grant codeBase "${es.security.insecure.plugin.discovery-ec2}" { + // needed because of problems in aws-sdk + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; + +grant codeBase "${es.security.insecure.plugin.cloud-gce}" { + // needed because of problems in cloud-gce + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; //// test framework permissions. @@ -43,6 +64,22 @@ grant codeBase "${es.security.jar.lucene.core}" { grant codeBase "${es.security.jar.elasticsearch.securemock}" { // needed to support creation of mocks permission java.lang.RuntimePermission "reflectionFactoryAccess"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; + +grant codeBase "${es.security.jar.lucene.testframework}" { + // needed by RamUsageTester + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; + +grant codeBase "${es.security.jar.randomizedtesting.runner}" { + // optionally needed for access to private test methods (e.g. beforeClass) + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; + +grant codeBase "${es.security.jar.randomizedtesting.junit4}" { + // needed for gson serialization + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; //// Everything else: @@ -58,9 +95,6 @@ grant { // Allow read/write to all system properties permission java.util.PropertyPermission "*", "read,write"; - // needed by junit4's gson usage - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; - // needed by scripting engines, etc permission java.lang.RuntimePermission "createClassLoader"; diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapForTesting.java index e31fc7d8b35..2c769fd7732 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapForTesting.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapForTesting.java @@ -83,6 +83,13 @@ public class BootstrapForTesting { if (systemPropertyAsBoolean("tests.security.manager", true)) { try { Security.setCodebaseProperties(); + // if its an insecure plugin, its not easy to simulate here, since we don't have a real plugin install. + // we just do our best so unit testing can work. integration tests for such plugins are essential. + String artifact = System.getProperty("tests.artifact"); + String insecurePluginProp = Security.INSECURE_PLUGINS.get(artifact); + if (insecurePluginProp != null) { + System.setProperty(insecurePluginProp, "file:/-"); + } // initialize paths the same exact way as bootstrap. Permissions perms = new Permissions(); // add permissions to everything in classpath @@ -117,6 +124,15 @@ public class BootstrapForTesting { Policy.setPolicy(new ESPolicy(perms)); System.setSecurityManager(new TestSecurityManager()); Security.selfTest(); + + if (insecurePluginProp != null) { + // initialize the plugin class, in case it has one-time hacks (unit tests often won't do this) + String clazz = System.getProperty("tests.plugin.classname"); + if (clazz == null) { + throw new IllegalStateException("plugin classname is needed for insecure plugin unit tests"); + } + Class.forName(clazz); + } } catch (Exception e) { throw new RuntimeException("unable to install test security manager", e); } diff --git a/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 41febf78e08..00000000000 --- a/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8243b938b75818e86aa8d270d8d99529e1847578 diff --git a/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..961bc6ff551 --- /dev/null +++ b/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +5792c4b38aa2cf1f66c9dea8bf139907e33fa018 diff --git a/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 17f6316e8dd..00000000000 --- a/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ba85c6e5e77e1f76c52c31d34a59558afa135d47 diff --git a/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..55354f421a5 --- /dev/null +++ b/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +cd6fa25bc29718d8c964b0734fc9a009547453db diff --git a/distribution/licenses/lucene-core-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-core-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index e6198d478b4..00000000000 --- a/distribution/licenses/lucene-core-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f8a38658b6393015c9b33c16b1b4122167b526b2 diff --git a/distribution/licenses/lucene-core-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-core-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..938d27bd3cf --- /dev/null +++ b/distribution/licenses/lucene-core-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +e4e0076ce4331309d1270a9c5b2edb51915fe32a diff --git a/distribution/licenses/lucene-expressions-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-expressions-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index ff8fd3666a0..00000000000 --- a/distribution/licenses/lucene-expressions-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fa5d27ecadbe346caaf5a571ba71944b51761acf diff --git a/distribution/licenses/lucene-expressions-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-expressions-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..d04bca5b6e4 --- /dev/null +++ b/distribution/licenses/lucene-expressions-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +ecfb9a923b19fac61b4e9a79275f6bd242b1f091 diff --git a/distribution/licenses/lucene-grouping-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-grouping-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 7acdc72f103..00000000000 --- a/distribution/licenses/lucene-grouping-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2c1464fcf6ede7819f8ba434b9bc7c79f5968407 diff --git a/distribution/licenses/lucene-grouping-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-grouping-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..98041824d8d --- /dev/null +++ b/distribution/licenses/lucene-grouping-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +49992ef742b6d3a24b551e06c96c0ab9cbad21e7 diff --git a/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 9f46054da43..00000000000 --- a/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a40f9a3ef224bc042ef2ad1b713e318911b6057a diff --git a/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..7c9ca048217 --- /dev/null +++ b/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +85544da78c2d33b2fdfa6f76eb621c8c963eae37 diff --git a/distribution/licenses/lucene-join-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-join-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 01a2d77b40f..00000000000 --- a/distribution/licenses/lucene-join-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0a7642c9b98cb3d9013fb33be5c0751baf9f0b31 diff --git a/distribution/licenses/lucene-join-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-join-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..ed8c024b716 --- /dev/null +++ b/distribution/licenses/lucene-join-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +5c8b58d902a01cfce046e656eddee5c4e0578316 diff --git a/distribution/licenses/lucene-memory-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-memory-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 6f9b8b4ecf1..00000000000 --- a/distribution/licenses/lucene-memory-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a0d6461ab9cda93ea530560b0c074a28fe0dd717 diff --git a/distribution/licenses/lucene-memory-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-memory-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..3edae47a468 --- /dev/null +++ b/distribution/licenses/lucene-memory-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +55202617e88437dd6def4e42ceb42d18d08a9f6e diff --git a/distribution/licenses/lucene-misc-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-misc-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index f274d1264de..00000000000 --- a/distribution/licenses/lucene-misc-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -85c5c7b78715c50157700c90ffd101537446533d diff --git a/distribution/licenses/lucene-misc-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-misc-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..ce5a5e7d435 --- /dev/null +++ b/distribution/licenses/lucene-misc-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +50b8439558061d0bbf09ddf8144d769143d33f00 diff --git a/distribution/licenses/lucene-queries-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-queries-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 295a311e3be..00000000000 --- a/distribution/licenses/lucene-queries-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -70ca782d6ed458b5f777141353e09600083ed4fe diff --git a/distribution/licenses/lucene-queries-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-queries-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..938aa1d45a0 --- /dev/null +++ b/distribution/licenses/lucene-queries-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +ce470e38912676ebc63838635663f17d8db844f5 diff --git a/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 88c548a7690..00000000000 --- a/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b4832cdfe7a6cc7c586a3e28d7cd530acb182232 diff --git a/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..25390b153da --- /dev/null +++ b/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +318e192d61eb28de09cc62c42d4e9d044a497e8b diff --git a/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 54f5944099d..00000000000 --- a/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bde73ae2b2324e1576c5789a7e6dd88b6543b939 diff --git a/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..25b114cc5ef --- /dev/null +++ b/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +043b3c3c03a5e0687884e5894754933d7e8f2f3e diff --git a/distribution/licenses/lucene-spatial-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-spatial-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index f3aa3630484..00000000000 --- a/distribution/licenses/lucene-spatial-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8d261ff1c2333ce1e040c3aefca9784d1ae71acc diff --git a/distribution/licenses/lucene-spatial-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-spatial-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..823fe3ab4f5 --- /dev/null +++ b/distribution/licenses/lucene-spatial-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +a9ab07d808456d9a2de248f3b4eba9765808b1cb diff --git a/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index d23a2710c0a..00000000000 --- a/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ee041e52dfcdb33a1aa6fab112042b5f33fc0c0c diff --git a/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..84e38b7424d --- /dev/null +++ b/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +c9ae48e40c89364e0e69f27c514c924d52e0d57e diff --git a/distribution/licenses/lucene-suggest-5.4.0-snapshot-1702265.jar.sha1 b/distribution/licenses/lucene-suggest-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index f2307a3bc86..00000000000 --- a/distribution/licenses/lucene-suggest-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a8ceb11b26e53612eee9a265ff454351f6dc99f2 diff --git a/distribution/licenses/lucene-suggest-5.4.0-snapshot-1702855.jar.sha1 b/distribution/licenses/lucene-suggest-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..e2ba8be0c55 --- /dev/null +++ b/distribution/licenses/lucene-suggest-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +8bd8fb7e75746fd7bce00b4d2813fca738d68f0b diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1702265.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 54c8e962a9d..00000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1f92d0376ca9219b0bf96fe5bd9a913089608d6a diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1702855.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..8d39139d91c --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +a71d7e2780ae064a6e6ea5b43357d070351620b8 diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1702265.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index a3885bfaaa1..00000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -60ee5bc1ac8ec102434e7064141a1f40281918b5 diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1702855.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..69991e5f6ad --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +a82b7a125e7cc16c6eb050b68bafc9f1e63eb646 diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1702265.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 44ac92c5195..00000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -977aa506485d358b40602347c11238b0f912fe2c diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1702855.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..b135301e7db --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +64b4db89ab7612284b5c685769c3550fb3018bbc diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1702265.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index 6f7d174c750..00000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -61911b8400160bd206ea6ea46ba08fd9ba09e72b diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1702855.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..a32957d25e3 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +a79e5ed1671d9f511c5aff273ba287557f5f9103 diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1702265.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1702265.jar.sha1 deleted file mode 100644 index cf50fb6450e..00000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1702265.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a9bdf48b63562bf1ac8a73c1c6bdb4cc450439e diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1702855.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1702855.jar.sha1 new file mode 100644 index 00000000000..9421876bf2f --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1702855.jar.sha1 @@ -0,0 +1 @@ +7b0b74aebbbfdd2175d6ce045fc538261e016417 diff --git a/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index c083f10baae..f89f6bc19f2 100644 --- a/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -27,6 +27,7 @@ import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.compute.Compute; import com.google.api.services.compute.model.Instance; import com.google.api.services.compute.model.InstanceList; + import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; @@ -34,7 +35,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; +import java.security.AccessController; import java.security.GeneralSecurityException; +import java.security.PrivilegedExceptionAction; import java.util.*; /** @@ -122,7 +125,15 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent() { + @Override + public Void run() throws IOException { + credential.refreshToken(); + return null; + } + }); logger.debug("token [{}] will expire in [{}] s", credential.getAccessToken(), credential.getExpiresInSeconds()); if (credential.getExpiresInSeconds() != null) { diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java index 38c704249d5..9ceb1cd34e7 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java @@ -28,6 +28,8 @@ import org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider; import org.elasticsearch.discovery.ec2.Ec2Discovery; import org.elasticsearch.plugins.Plugin; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collection; @@ -35,6 +37,23 @@ import java.util.Collection; * */ public class Ec2DiscoveryPlugin extends Plugin { + + static { + // This internal config is deserialized but with wrong access modifiers, + // cannot work without suppressAccessChecks permission right now. We force + // a one time load with elevated privileges as a workaround. + AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Void run() { + try { + Class.forName("com.amazonaws.internal.config.InternalConfig$Factory"); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Unable to initialize internal aws config", e); + } + return null; + } + }); + } @Override public String name() { diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java index 2911e278c38..7f4f663ce36 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java @@ -27,6 +27,8 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.repositories.s3.S3Repository; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -35,6 +37,23 @@ import java.util.Collections; * */ public class S3RepositoryPlugin extends Plugin { + + static { + // This internal config is deserialized but with wrong access modifiers, + // cannot work without suppressAccessChecks permission right now. We force + // a one time load with elevated privileges as a workaround. + AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Void run() { + try { + Class.forName("com.amazonaws.internal.config.InternalConfig$Factory"); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Unable to initialize internal aws config", e); + } + return null; + } + }); + } @Override public String name() { diff --git a/pom.xml b/pom.xml index ceef389cd37..54a6d7cd26b 100644 --- a/pom.xml +++ b/pom.xml @@ -45,9 +45,14 @@ 5.4.0 - 1702265 + 1702855 5.4.0-snapshot-${lucene.snapshot.revision} - 2.1.16 + + pr202 + 2.2.0-snapshot-${testframework.snapshot.revision} 2.5.3 1.6.2 1.2.17 @@ -145,8 +150,19 @@ Lucene snapshots https://download.elasticsearch.org/lucenesnapshots/${lucene.snapshot.revision} + + Randomized Runner snapshot + https://download.elasticsearch.org/lucenesnapshots/rr-${testframework.snapshot.revision} + + + + Randomized Runner snapshot + https://download.elasticsearch.org/lucenesnapshots/rr-${testframework.snapshot.revision} + + + @@ -633,6 +649,9 @@ ${tests.cluster} ${tests.iters} ${project.groupId}:${project.artifactId} + + ${project.artifactId} + ${elasticsearch.plugin.classname} ${tests.maxfailures} ${tests.failfast} ${tests.class} From 51773f2a66101c9eb64618ca044634e68d1bb441 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Wed, 16 Sep 2015 04:31:27 -0400 Subject: [PATCH 30/33] fix test to expect AssertionError as well --- .../org/elasticsearch/index/engine/InternalEngineTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 3381102a419..ba65a1026ae 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -1728,7 +1728,9 @@ public class InternalEngineTests extends ESTestCase { engine = createEngine(store, primaryTranslogDir); started = true; break; - } catch (EngineCreationFailureException ex) { + } catch (EngineCreationFailureException | AssertionError ex) { + // IndexWriter can throw AssertionError on init (if asserts are enabled) if we throw FNFE/NSFE when it asserts that all + // referenced files in the current commit point do exist } } From 30a779511993646db704de45020b79d5efcb2ea8 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 16 Sep 2015 04:50:53 -0400 Subject: [PATCH 31/33] Remove unused import in o.e.c.u.i.Iterables --- .../java/org/elasticsearch/common/util/iterable/Iterables.java | 2 -- .../org/elasticsearch/index/mapper/FieldTypeLookupTests.java | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java b/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java index f6b891c9997..76ba12bafe9 100644 --- a/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java +++ b/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java @@ -19,8 +19,6 @@ package org.elasticsearch.common.util.iterable; -import org.elasticsearch.common.lucene.store.IndexOutputOutputStream; - import java.util.*; import java.util.stream.Stream; import java.util.stream.StreamSupport; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index 41a44d4a88c..5e3b61a7da1 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -30,6 +30,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.stream.StreamSupport; public class FieldTypeLookupTests extends ESTestCase { From 0060a749cac02b3401de8879871cae38b7c7db12 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Wed, 16 Sep 2015 04:58:16 -0400 Subject: [PATCH 32/33] Fix test case to verify the AssertionError did in fact come from IndexWriter.filesExist --- .../elasticsearch/index/engine/InternalEngineTests.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index ba65a1026ae..818e2b08122 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -41,6 +41,7 @@ import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.TestUtil; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -1728,9 +1729,11 @@ public class InternalEngineTests extends ESTestCase { engine = createEngine(store, primaryTranslogDir); started = true; break; - } catch (EngineCreationFailureException | AssertionError ex) { - // IndexWriter can throw AssertionError on init (if asserts are enabled) if we throw FNFE/NSFE when it asserts that all - // referenced files in the current commit point do exist + } catch (EngineCreationFailureException ex) { + } catch (AssertionError ex) { + // IndexWriter can throw AssertionError on init (if asserts are enabled) if our directory randomly throws FNFE/NSFE when + // it asserts that all referenced files in the current commit point do exist + assertTrue(ExceptionsHelper.stackTrace(ex).contains("org.apache.lucene.index.IndexWriter.filesExist")); } } From 3f40e1ca3c6166d4ce36d4ef34539286845e21fa Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Wed, 26 Aug 2015 18:08:03 +0200 Subject: [PATCH 33/33] make sure JAVA_HOME is set before tests are run relates to #12961 --- pom.xml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ceef389cd37..ca10b996f33 100644 --- a/pom.xml +++ b/pom.xml @@ -462,6 +462,7 @@ org.apache.maven.plugins maven-enforcer-plugin + 1.4.1 org.apache.maven.plugins @@ -526,7 +527,6 @@ org.apache.maven.plugins maven-enforcer-plugin - 1.4.1 enforce-versions @@ -561,6 +561,21 @@ display-info + + enforce-java-home-is-set + + enforce + + + + + JAVA_HOME + "JAVA_HOME must be set and point to the jdk to run the tests" + + + true + +