diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bbb01426fde..cc0d2ba6e35 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -141,7 +141,7 @@ Please follow these formatting guidelines: * Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do. * Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause the build to fail. Please attempt to tame your IDE so it doesn't make them and please send a PR against this document with instructions for your IDE if it doesn't contain them. * Eclipse: `Preferences->Java->Code Style->Organize Imports`. There are two boxes labeled "`Number of (static )? imports needed for .*`". Set their values to 99999 or some other absurdly high value. - * IntelliJ: `Preferences->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value. + * IntelliJ: `Preferences/Settings->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value. * Don't worry too much about import order. Try not to change it but don't worry about fighting your IDE to stop it from doing so. To create a distribution from the source, simply run: diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 0f66a5fca9c..03a682f62fa 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -66,6 +66,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AsyncIOProcessor; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexModule; @@ -2421,7 +2422,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl indexSettings::getMaxRefreshListeners, () -> refresh("too_many_listeners"), threadPool.executor(ThreadPool.Names.LISTENER)::execute, - logger); + logger, threadPool.getThreadContext()); } /** diff --git a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java index f0df6e12b8c..17e824eb046 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java +++ b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.shard; import org.apache.logging.log4j.Logger; import org.apache.lucene.search.ReferenceManager; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.translog.Translog; import java.io.Closeable; @@ -45,6 +46,7 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener, private final Runnable forceRefresh; private final Executor listenerExecutor; private final Logger logger; + private final ThreadContext threadContext; /** * Is this closed? If true then we won't add more listeners and have flushed all pending listeners. @@ -63,11 +65,13 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener, */ private volatile Translog.Location lastRefreshedLocation; - public RefreshListeners(IntSupplier getMaxRefreshListeners, Runnable forceRefresh, Executor listenerExecutor, Logger logger) { + public RefreshListeners(IntSupplier getMaxRefreshListeners, Runnable forceRefresh, Executor listenerExecutor, Logger logger, + ThreadContext threadContext) { this.getMaxRefreshListeners = getMaxRefreshListeners; this.forceRefresh = forceRefresh; this.listenerExecutor = listenerExecutor; this.logger = logger; + this.threadContext = threadContext; } /** @@ -98,8 +102,15 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener, refreshListeners = listeners; } if (listeners.size() < getMaxRefreshListeners.getAsInt()) { + ThreadContext.StoredContext storedContext = threadContext.newStoredContext(true); + Consumer contextPreservingListener = forced -> { + try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { + storedContext.restore(); + listener.accept(forced); + } + }; // We have a free slot so register the listener - listeners.add(new Tuple<>(location, listener)); + listeners.add(new Tuple<>(location, contextPreservingListener)); return false; } } diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index fcc3c93fc37..53ced098c04 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -67,6 +68,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -87,16 +89,16 @@ public class RefreshListenersTests extends ESTestCase { public void setupListeners() throws Exception { // Setup dependencies of the listeners maxListeners = randomIntBetween(1, 1000); + // Now setup the InternalEngine which is much more complicated because we aren't mocking anything + threadPool = new TestThreadPool(getTestName()); listeners = new RefreshListeners( () -> maxListeners, () -> engine.refresh("too-many-listeners"), // Immediately run listeners rather than adding them to the listener thread pool like IndexShard does to simplify the test. Runnable::run, - logger - ); + logger, + threadPool.getThreadContext()); - // Now setup the InternalEngine which is much more complicated because we aren't mocking anything - threadPool = new TestThreadPool(getTestName()); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY); ShardId shardId = new ShardId(new Index("index", "_na_"), 1); String allocationId = UUIDs.randomBase64UUID(random()); @@ -161,6 +163,23 @@ public class RefreshListenersTests extends ESTestCase { assertEquals(0, listeners.pendingCount()); } + public void testContextIsPreserved() throws IOException, InterruptedException { + assertEquals(0, listeners.pendingCount()); + Engine.IndexResult index = index("1"); + CountDownLatch latch = new CountDownLatch(1); + try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { + threadPool.getThreadContext().putHeader("test", "foobar"); + assertFalse(listeners.addOrNotify(index.getTranslogLocation(), forced -> { + assertEquals("foobar", threadPool.getThreadContext().getHeader("test")); + latch.countDown(); + })); + } + assertNull(threadPool.getThreadContext().getHeader("test")); + assertEquals(1, latch.getCount()); + engine.refresh("I said so"); + latch.await(); + } + public void testTooMany() throws Exception { assertEquals(0, listeners.pendingCount()); assertFalse(listeners.refreshNeeded()); diff --git a/docs/reference/setup/sysconfig/threads.asciidoc b/docs/reference/setup/sysconfig/threads.asciidoc index 1cc65b7a03b..a9a75dc872b 100644 --- a/docs/reference/setup/sysconfig/threads.asciidoc +++ b/docs/reference/setup/sysconfig/threads.asciidoc @@ -4,9 +4,9 @@ Elasticsearch uses a number of thread pools for different types of operations. It is important that it is able to create new threads whenever needed. Make sure that the number of threads that the Elasticsearch user can create is at -least 2048. +least 4096. -This can be done by setting <> as root before -starting Elasticsearch, or by setting `nproc` to `2048` in +This can be done by setting <> as root before +starting Elasticsearch, or by setting `nproc` to `4096` in <>. diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java index 4aa36ba3714..3e1c2ff2db1 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java @@ -1076,9 +1076,11 @@ public final class Walker extends PainlessParserBaseVisitor { } } + FunctionReserved lambdaReserved = (FunctionReserved)reserved.pop(); + reserved.peek().addUsedVariables(lambdaReserved); + String name = nextLambda(); - return new ELambda(name, (FunctionReserved)reserved.pop(), location(ctx), - paramTypes, paramNames, statements); + return new ELambda(name, lambdaReserved, location(ctx), paramTypes, paramNames, statements); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java index 84c6145ac0c..873f109e72d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java @@ -20,7 +20,6 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Definition.Cast; import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Globals; @@ -213,6 +212,11 @@ public final class EAssignment extends AExpression { // If the lhs node is a def optimized node we update the actual type to remove the need for a cast. if (lhs.isDefOptimized()) { rhs.analyze(locals); + + if (rhs.actual.clazz == void.class) { + throw createError(new IllegalArgumentException("Right-hand side cannot be a [void] type for assignment.")); + } + rhs.expected = rhs.actual; lhs.updateActual(rhs.actual); // Otherwise, we must adapt the rhs type to the lhs type with a cast. diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java index 89fc169704f..560acaf131e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java @@ -19,9 +19,7 @@ package org.elasticsearch.painless.node; -import java.util.Collections; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -29,6 +27,7 @@ import org.elasticsearch.painless.MethodWriter; import org.objectweb.asm.Type; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Set; @@ -76,6 +75,10 @@ final class PSubDefCall extends AExpression { totalCaptures += lambda.getCaptureCount(); } + if (expression.actual.clazz == void.class) { + throw createError(new IllegalArgumentException("Argument(s) cannot be of [void] type when calling method [" + name + "].")); + } + expression.expected = expression.actual; arguments.set(argument, expression.cast(locals)); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java index c1f0c468e42..99d60c3b73f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java @@ -20,7 +20,6 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java index 6c479265cfe..6183697bbe4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java @@ -41,11 +41,13 @@ import java.lang.invoke.MethodType; import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; import static java.util.Collections.emptyList; +import static java.util.Collections.unmodifiableSet; import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE; /** @@ -53,11 +55,22 @@ import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE; */ public final class SFunction extends AStatement { public static final class FunctionReserved implements Reserved { + private final Set usedVariables = new HashSet<>(); private int maxLoopCounter = 0; @Override public void markUsedVariable(String name) { - // Do nothing. + usedVariables.add(name); + } + + @Override + public Set getUsedVariables() { + return unmodifiableSet(usedVariables); + } + + @Override + public void addUsedVariables(FunctionReserved reserved) { + usedVariables.addAll(reserved.getUsedVariables()); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java index 882b018bc41..c1ab8398bad 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java @@ -32,6 +32,7 @@ import org.elasticsearch.painless.MethodWriter; import org.elasticsearch.painless.ScriptClassInfo; import org.elasticsearch.painless.SimpleChecksAdapter; import org.elasticsearch.painless.WriterConstants; +import org.elasticsearch.painless.node.SFunction.FunctionReserved; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; import org.objectweb.asm.Label; @@ -89,6 +90,8 @@ public final class SSource extends AStatement { */ public interface Reserved { void markUsedVariable(String name); + Set getUsedVariables(); + void addUsedVariables(FunctionReserved reserved); void setMaxLoopCounter(int max); int getMaxLoopCounter(); @@ -103,6 +106,16 @@ public final class SSource extends AStatement { usedVariables.add(name); } + @Override + public Set getUsedVariables() { + return unmodifiableSet(usedVariables); + } + + @Override + public void addUsedVariables(FunctionReserved reserved) { + usedVariables.addAll(reserved.getUsedVariables()); + } + @Override public void setMaxLoopCounter(int max) { maxLoopCounter = max; @@ -112,10 +125,6 @@ public final class SSource extends AStatement { public int getMaxLoopCounter() { return maxLoopCounter; } - - public Set getUsedVariables() { - return unmodifiableSet(usedVariables); - } } private final ScriptClassInfo scriptClassInfo; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java index 554da280dda..f124d088bf2 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java @@ -22,7 +22,7 @@ package org.elasticsearch.painless; /** Tests for addition operator across all types */ //TODO: NaN/Inf/overflow/... public class AdditionTests extends ScriptTestCase { - + public void testBasics() throws Exception { assertEquals(3.0, exec("double x = 1; byte y = 2; return x + y;")); } @@ -195,7 +195,7 @@ public class AdditionTests extends ScriptTestCase { assertEquals(1.0+0.0, exec("return 1.0+0.0;")); assertEquals(0.0+0.0, exec("return 0.0+0.0;")); } - + public void testDef() { assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y")); assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x + y")); @@ -253,7 +253,7 @@ public class AdditionTests extends ScriptTestCase { assertEquals(2D, exec("def x = (float)1; def y = (double)1; return x + y")); assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y")); } - + public void testDefTypedLHS() { assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x + y")); assertEquals(2, exec("short x = (short)1; def y = (byte)1; return x + y")); @@ -311,7 +311,7 @@ public class AdditionTests extends ScriptTestCase { assertEquals(2D, exec("float x = (float)1; def y = (double)1; return x + y")); assertEquals(2D, exec("double x = (double)1; def y = (double)1; return x + y")); } - + public void testDefTypedRHS() { assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x + y")); assertEquals(2, exec("def x = (short)1; byte y = (byte)1; return x + y")); @@ -369,19 +369,19 @@ public class AdditionTests extends ScriptTestCase { assertEquals(2D, exec("def x = (float)1; double y = (double)1; return x + y")); assertEquals(2D, exec("def x = (double)1; double y = (double)1; return x + y")); } - + public void testDefNulls() { expectScriptThrows(NullPointerException.class, () -> { - exec("def x = null; int y = 1; return x + y"); + exec("def x = null; int y = 1; return x + y"); }); expectScriptThrows(NullPointerException.class, () -> { - exec("int x = 1; def y = null; return x + y"); + exec("int x = 1; def y = null; return x + y"); }); expectScriptThrows(NullPointerException.class, () -> { - exec("def x = null; def y = 1; return x + y"); + exec("def x = null; def y = 1; return x + y"); }); } - + public void testCompoundAssignment() { // byte assertEquals((byte) 15, exec("byte x = 5; x += 10; return x;")); @@ -406,7 +406,7 @@ public class AdditionTests extends ScriptTestCase { assertEquals(15D, exec("double x = 5.0; x += 10; return x;")); assertEquals(-5D, exec("double x = 5.0; x += -10; return x;")); } - + public void testDefCompoundAssignmentLHS() { // byte assertEquals((byte) 15, exec("def x = (byte)5; x += 10; return x;")); @@ -431,7 +431,7 @@ public class AdditionTests extends ScriptTestCase { assertEquals(15D, exec("def x = 5.0; x += 10; return x;")); assertEquals(-5D, exec("def x = 5.0; x += -10; return x;")); } - + public void testDefCompoundAssignmentRHS() { // byte assertEquals((byte) 15, exec("byte x = 5; def y = 10; x += y; return x;")); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java index 0ca72f993e5..c9954fd7171 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java @@ -318,4 +318,13 @@ public class CastTests extends ScriptTestCase { exec("def x = 5L; boolean y = (boolean) (x + x); return y"); }); } + + public void testIllegalVoidCasts() { + expectScriptThrows(IllegalArgumentException.class, () -> { + exec("def map = ['a': 1,'b': 2,'c': 3]; map.c = Collections.sort(new ArrayList(map.keySet()));"); + }); + expectScriptThrows(IllegalArgumentException.class, () -> { + exec("Map map = ['a': 1,'b': 2,'c': 3]; def x = new HashMap(); x.put(1, map.clear());"); + }); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java index 23362265474..b15a2747bd0 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java @@ -191,4 +191,13 @@ public class FactoryTests extends ScriptTestCase { assertEquals("def", script.execute()); assertEquals("def", script.execute()); } + + public void testGetterInLambda() { + FactoryTestScript.Factory factory = + scriptEngine.compile("template_test", + "IntSupplier createLambda(IntSupplier s) { return s; } createLambda(() -> params['x'] + test).getAsInt()", + FactoryTestScript.CONTEXT, Collections.emptyMap()); + FactoryTestScript script = factory.newInstance(Collections.singletonMap("x", 1)); + assertEquals(2, script.execute(1)); + } }