honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into ccr 

* master:
  Fix max number of threads bootstrap docs
  Clarify setting IntelliJ preferences
  Painless: Fix errors allowing void to be assigned to def. (#27460)
  Painless: Fix variable scoping issue in lambdas not including captured variables. (#27571)
  Ensure threadcontext is preserved when refresh listeners are invoked (#27565)
This commit is contained in:
Jason Tedor 2017-11-28 22:34:53 -05:00
parent d803d5562d 23f85fe6d4
commit d2841d4e7d
14 changed files with 112 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CONTRIBUTING.md
View File

@ -141,7 +141,7 @@ Please follow these formatting guidelines:
* Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do.
* Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause the build to fail. Please attempt to tame your IDE so it doesn't make them and please send a PR against this document with instructions for your IDE if it doesn't contain them.
* Eclipse: `Preferences->Java->Code Style->Organize Imports`. There are two boxes labeled "`Number of (static )? imports needed for .*`". Set their values to 99999 or some other absurdly high value.
* IntelliJ: `Preferences->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value.
* IntelliJ: `Preferences/Settings->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value.
* Don't worry too much about import order. Try not to change it but don't worry about fighting your IDE to stop it from doing so.
To create a distribution from the source, simply run:

3
core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
View File

@ -66,6 +66,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.util.concurrent.AsyncIOProcessor;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexModule;
@ -2421,7 +2422,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
indexSettings::getMaxRefreshListeners,
() -> refresh("too_many_listeners"),
threadPool.executor(ThreadPool.Names.LISTENER)::execute,
logger);
logger, threadPool.getThreadContext());
}
/**

15
core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.index.shard;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.search.ReferenceManager;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.index.translog.Translog;
import java.io.Closeable;
@ -45,6 +46,7 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener,
private final Runnable forceRefresh;
private final Executor listenerExecutor;
private final Logger logger;
private final ThreadContext threadContext;
/**
* Is this closed? If true then we won't add more listeners and have flushed all pending listeners.
@ -63,11 +65,13 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener,
*/
private volatile Translog.Location lastRefreshedLocation;
public RefreshListeners(IntSupplier getMaxRefreshListeners, Runnable forceRefresh, Executor listenerExecutor, Logger logger) {
public RefreshListeners(IntSupplier getMaxRefreshListeners, Runnable forceRefresh, Executor listenerExecutor, Logger logger,
ThreadContext threadContext) {
this.getMaxRefreshListeners = getMaxRefreshListeners;
this.forceRefresh = forceRefresh;
this.listenerExecutor = listenerExecutor;
this.logger = logger;
this.threadContext = threadContext;
}
/**
@ -98,8 +102,15 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener,
refreshListeners = listeners;
}
if (listeners.size() < getMaxRefreshListeners.getAsInt()) {
ThreadContext.StoredContext storedContext = threadContext.newStoredContext(true);
Consumer<Boolean> contextPreservingListener = forced -> {
try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
storedContext.restore();
listener.accept(forced);
}
};
// We have a free slot so register the listener
listeners.add(new Tuple<>(location, listener));
listeners.add(new Tuple<>(location, contextPreservingListener));
return false;
}
}

27
core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
View File

@ -35,6 +35,7 @@ import org.elasticsearch.common.lucene.uid.Versions;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
@ -67,6 +68,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
@ -87,16 +89,16 @@ public class RefreshListenersTests extends ESTestCase {
public void setupListeners() throws Exception {
// Setup dependencies of the listeners
maxListeners = randomIntBetween(1, 1000);
// Now setup the InternalEngine which is much more complicated because we aren't mocking anything
threadPool = new TestThreadPool(getTestName());
listeners = new RefreshListeners(
() -> maxListeners,
() -> engine.refresh("too-many-listeners"),
// Immediately run listeners rather than adding them to the listener thread pool like IndexShard does to simplify the test.
Runnable::run,
logger
);
logger,
threadPool.getThreadContext());
// Now setup the InternalEngine which is much more complicated because we aren't mocking anything
threadPool = new TestThreadPool(getTestName());
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY);
ShardId shardId = new ShardId(new Index("index", "_na_"), 1);
String allocationId = UUIDs.randomBase64UUID(random());
@ -161,6 +163,23 @@ public class RefreshListenersTests extends ESTestCase {
assertEquals(0, listeners.pendingCount());
}
public void testContextIsPreserved() throws IOException, InterruptedException {
assertEquals(0, listeners.pendingCount());
Engine.IndexResult index = index("1");
CountDownLatch latch = new CountDownLatch(1);
try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) {
threadPool.getThreadContext().putHeader("test", "foobar");
assertFalse(listeners.addOrNotify(index.getTranslogLocation(), forced -> {
assertEquals("foobar", threadPool.getThreadContext().getHeader("test"));
latch.countDown();
}));
}
assertNull(threadPool.getThreadContext().getHeader("test"));
assertEquals(1, latch.getCount());
engine.refresh("I said so");
latch.await();
}
public void testTooMany() throws Exception {
assertEquals(0, listeners.pendingCount());
assertFalse(listeners.refreshNeeded());

6
docs/reference/setup/sysconfig/threads.asciidoc
View File

@ -4,9 +4,9 @@
Elasticsearch uses a number of thread pools for different types of operations.
It is important that it is able to create new threads whenever needed. Make
sure that the number of threads that the Elasticsearch user can create is at
least 2048.
least 4096.
This can be done by setting <<ulimit,`ulimit -u 2048`>> as root before
starting Elasticsearch, or by setting `nproc` to `2048` in
This can be done by setting <<ulimit,`ulimit -u 4096`>> as root before
starting Elasticsearch, or by setting `nproc` to `4096` in
<<limits.conf,`/etc/security/limits.conf`>>.

6
modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
View File

@ -1076,9 +1076,11 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
}
}
FunctionReserved lambdaReserved = (FunctionReserved)reserved.pop();
reserved.peek().addUsedVariables(lambdaReserved);
String name = nextLambda();
return new ELambda(name, (FunctionReserved)reserved.pop(), location(ctx),
paramTypes, paramNames, statements);
return new ELambda(name, lambdaReserved, location(ctx), paramTypes, paramNames, statements);
}
@Override

6
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.painless.node;
import org.elasticsearch.painless.DefBootstrap;
import org.elasticsearch.painless.Definition;
import org.elasticsearch.painless.Definition.Cast;
import org.elasticsearch.painless.Definition.Type;
import org.elasticsearch.painless.Globals;
@ -213,6 +212,11 @@ public final class EAssignment extends AExpression {
// If the lhs node is a def optimized node we update the actual type to remove the need for a cast.
if (lhs.isDefOptimized()) {
rhs.analyze(locals);
if (rhs.actual.clazz == void.class) {
throw createError(new IllegalArgumentException("Right-hand side cannot be a [void] type for assignment."));
}
rhs.expected = rhs.actual;
lhs.updateActual(rhs.actual);
// Otherwise, we must adapt the rhs type to the lhs type with a cast.

7
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java
View File

@ -19,9 +19,7 @@
package org.elasticsearch.painless.node;
import java.util.Collections;
import org.elasticsearch.painless.DefBootstrap;
import org.elasticsearch.painless.Definition;
import org.elasticsearch.painless.Globals;
import org.elasticsearch.painless.Locals;
import org.elasticsearch.painless.Location;
@ -29,6 +27,7 @@ import org.elasticsearch.painless.MethodWriter;
import org.objectweb.asm.Type;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Set;
@ -76,6 +75,10 @@ final class PSubDefCall extends AExpression {
totalCaptures += lambda.getCaptureCount();
}
if (expression.actual.clazz == void.class) {
throw createError(new IllegalArgumentException("Argument(s) cannot be of [void] type when calling method [" + name + "]."));
}
expression.expected = expression.actual;
arguments.set(argument, expression.cast(locals));
}

1
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.painless.node;
import org.elasticsearch.painless.DefBootstrap;
import org.elasticsearch.painless.Definition;
import org.elasticsearch.painless.Definition.Type;
import org.elasticsearch.painless.Globals;
import org.elasticsearch.painless.Locals;

15
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
View File

@ -41,11 +41,13 @@ import java.lang.invoke.MethodType;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import static java.util.Collections.emptyList;
import static java.util.Collections.unmodifiableSet;
import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
/**
@ -53,11 +55,22 @@ import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
*/
public final class SFunction extends AStatement {
public static final class FunctionReserved implements Reserved {
private final Set<String> usedVariables = new HashSet<>();
private int maxLoopCounter = 0;
@Override
public void markUsedVariable(String name) {
// Do nothing.
usedVariables.add(name);
}
@Override
public Set<String> getUsedVariables() {
return unmodifiableSet(usedVariables);
}
@Override
public void addUsedVariables(FunctionReserved reserved) {
usedVariables.addAll(reserved.getUsedVariables());
}
@Override

17
modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.painless.MethodWriter;
import org.elasticsearch.painless.ScriptClassInfo;
import org.elasticsearch.painless.SimpleChecksAdapter;
import org.elasticsearch.painless.WriterConstants;
import org.elasticsearch.painless.node.SFunction.FunctionReserved;
import org.objectweb.asm.ClassVisitor;
import org.objectweb.asm.ClassWriter;
import org.objectweb.asm.Label;
@ -89,6 +90,8 @@ public final class SSource extends AStatement {
*/
public interface Reserved {
void markUsedVariable(String name);
Set<String> getUsedVariables();
void addUsedVariables(FunctionReserved reserved);
void setMaxLoopCounter(int max);
int getMaxLoopCounter();
@ -103,6 +106,16 @@ public final class SSource extends AStatement {
usedVariables.add(name);
}
@Override
public Set<String> getUsedVariables() {
return unmodifiableSet(usedVariables);
}
@Override
public void addUsedVariables(FunctionReserved reserved) {
usedVariables.addAll(reserved.getUsedVariables());
}
@Override
public void setMaxLoopCounter(int max) {
maxLoopCounter = max;
@ -112,10 +125,6 @@ public final class SSource extends AStatement {
public int getMaxLoopCounter() {
return maxLoopCounter;
}
public Set<String> getUsedVariables() {
return unmodifiableSet(usedVariables);
}
}
private final ScriptClassInfo scriptClassInfo;

22
modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
View File

@ -22,7 +22,7 @@ package org.elasticsearch.painless;
/** Tests for addition operator across all types */
//TODO: NaN/Inf/overflow/...
public class AdditionTests extends ScriptTestCase {
public void testBasics() throws Exception {
assertEquals(3.0, exec("double x = 1; byte y = 2; return x + y;"));
}
@ -195,7 +195,7 @@ public class AdditionTests extends ScriptTestCase {
assertEquals(1.0+0.0, exec("return 1.0+0.0;"));
assertEquals(0.0+0.0, exec("return 0.0+0.0;"));
}
public void testDef() {
assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y"));
assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x + y"));
@ -253,7 +253,7 @@ public class AdditionTests extends ScriptTestCase {
assertEquals(2D, exec("def x = (float)1; def y = (double)1; return x + y"));
assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y"));
}
public void testDefTypedLHS() {
assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x + y"));
assertEquals(2, exec("short x = (short)1; def y = (byte)1; return x + y"));
@ -311,7 +311,7 @@ public class AdditionTests extends ScriptTestCase {
assertEquals(2D, exec("float x = (float)1; def y = (double)1; return x + y"));
assertEquals(2D, exec("double x = (double)1; def y = (double)1; return x + y"));
}
public void testDefTypedRHS() {
assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x + y"));
assertEquals(2, exec("def x = (short)1; byte y = (byte)1; return x + y"));
@ -369,19 +369,19 @@ public class AdditionTests extends ScriptTestCase {
assertEquals(2D, exec("def x = (float)1; double y = (double)1; return x + y"));
assertEquals(2D, exec("def x = (double)1; double y = (double)1; return x + y"));
}
public void testDefNulls() {
expectScriptThrows(NullPointerException.class, () -> {
exec("def x = null; int y = 1; return x + y");
exec("def x = null; int y = 1; return x + y");
});
expectScriptThrows(NullPointerException.class, () -> {
exec("int x = 1; def y = null; return x + y");
exec("int x = 1; def y = null; return x + y");
});
expectScriptThrows(NullPointerException.class, () -> {
exec("def x = null; def y = 1; return x + y");
exec("def x = null; def y = 1; return x + y");
});
}
public void testCompoundAssignment() {
// byte
assertEquals((byte) 15, exec("byte x = 5; x += 10; return x;"));
@ -406,7 +406,7 @@ public class AdditionTests extends ScriptTestCase {
assertEquals(15D, exec("double x = 5.0; x += 10; return x;"));
assertEquals(-5D, exec("double x = 5.0; x += -10; return x;"));
}
public void testDefCompoundAssignmentLHS() {
// byte
assertEquals((byte) 15, exec("def x = (byte)5; x += 10; return x;"));
@ -431,7 +431,7 @@ public class AdditionTests extends ScriptTestCase {
assertEquals(15D, exec("def x = 5.0; x += 10; return x;"));
assertEquals(-5D, exec("def x = 5.0; x += -10; return x;"));
}
public void testDefCompoundAssignmentRHS() {
// byte
assertEquals((byte) 15, exec("byte x = 5; def y = 10; x += y; return x;"));

9
modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java
View File

@ -318,4 +318,13 @@ public class CastTests extends ScriptTestCase {
exec("def x = 5L; boolean y = (boolean) (x + x); return y");
});
}
public void testIllegalVoidCasts() {
expectScriptThrows(IllegalArgumentException.class, () -> {
exec("def map = ['a': 1,'b': 2,'c': 3]; map.c = Collections.sort(new ArrayList(map.keySet()));");
});
expectScriptThrows(IllegalArgumentException.class, () -> {
exec("Map map = ['a': 1,'b': 2,'c': 3]; def x = new HashMap(); x.put(1, map.clear());");
});
}
}

9
modules/lang-painless/src/test/java/org/elasticsearch/painless/FactoryTests.java
View File

@ -191,4 +191,13 @@ public class FactoryTests extends ScriptTestCase {
assertEquals("def", script.execute());
assertEquals("def", script.execute());
}
public void testGetterInLambda() {
FactoryTestScript.Factory factory =
scriptEngine.compile("template_test",
"IntSupplier createLambda(IntSupplier s) { return s; } createLambda(() -> params['x'] + test).getAsInt()",
FactoryTestScript.CONTEXT, Collections.emptyMap());
FactoryTestScript script = factory.newInstance(Collections.singletonMap("x", 1));
assertEquals(2, script.execute(1));
}
}