honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/query-refactoring

This commit is contained in:
Christoph Büscher 2015-08-24 14:36:46 +02:00
parent 37b2007eb4 2336da1704
commit 8757af2d92
52 changed files with 452 additions and 428 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
View File

@ -21,15 +21,12 @@ package org.elasticsearch.bootstrap;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.StringHelper;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.Version;
import org.elasticsearch.common.PidFile;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.cli.CliTool;
import org.elasticsearch.common.cli.Terminal;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.CreationException;
import org.elasticsearch.common.inject.spi.Message;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
@ -44,16 +41,14 @@ import org.elasticsearch.node.NodeBuilder;
import org.elasticsearch.node.internal.InternalSettingsPreparer;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import static com.google.common.collect.Sets.newHashSet;
import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
/**
* A main entry point when starting from the command line.
* Internal startup code.
*/
public class Bootstrap {
final class Bootstrap {
private static volatile Bootstrap INSTANCE;
@ -137,10 +132,6 @@ public class Bootstrap {
OsProbe.getInstance();
}
public static boolean isMemoryLocked() {
return Natives.isMemoryLocked();
}
private void setup(boolean addShutdownHook, Settings settings, Environment environment) throws Exception {
initializeNatives(settings.getAsBoolean("bootstrap.mlockall", false),
settings.getAsBoolean("bootstrap.ctrlhandler", true));
@ -222,7 +213,11 @@ public class Bootstrap {
}
}
public static void main(String[] args) throws Throwable {
/**
* This method is invoked by {@link Elasticsearch#main(String[])}
* to startup elasticsearch.
*/
static void init(String[] args) throws Throwable {
BootstrapCLIParser bootstrapCLIParser = new BootstrapCLIParser();
CliTool.ExitStatus status = bootstrapCLIParser.execute(args);
@ -291,7 +286,7 @@ public class Bootstrap {
Loggers.enableConsoleLogging();
}
throw new StartupError(e);
throw e;
}
}

2
core/src/main/java/org/elasticsearch/bootstrap/BootstrapCLIParser.java
View File

@ -38,7 +38,7 @@ import java.util.Properties;
import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd;
import static org.elasticsearch.common.cli.CliToolConfig.Builder.optionBuilder;
public class BootstrapCLIParser extends CliTool {
final class BootstrapCLIParser extends CliTool {
private static final CliToolConfig CONFIG = CliToolConfig.config("elasticsearch", BootstrapCLIParser.class)
.cmds(Start.CMD, Version.CMD)

36
core/src/main/java/org/elasticsearch/common/inject/SpawnModules.java → core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java
View File

@ -17,20 +17,30 @@
* under the License.
*/
package org.elasticsearch.common.inject;
package org.elasticsearch.bootstrap;
/**
* This interface can be added to a Module to spawn sub modules. DO NOT USE.
*
* This is fundamentally broken.
* <ul>
* <li>If you have a plugin with multiple modules, return all the modules at once.</li>
* <li>If you are trying to make the implementation of a module "pluggable", don't do it.
* This is not extendable because custom implementations (using onModule) cannot be
* registered before spawnModules() is called.</li>
* </ul>
/**
* Exposes system startup information
*/
public interface SpawnModules {
public final class BootstrapInfo {
Iterable<? extends Module> spawnModules();
/** no instantiation */
private BootstrapInfo() {}
/**
* Returns true if we successfully loaded native libraries.
* <p>
* If this returns false, then native operations such as locking
* memory did not work.
*/
public static boolean isNativesAvailable() {
return Natives.JNA_AVAILABLE;
}
/**
* Returns true if we were able to lock the process's address space.
*/
public static boolean isMemoryLocked() {
return Natives.isMemoryLocked();
}
}

20
core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
View File

@ -20,11 +20,23 @@
package org.elasticsearch.bootstrap;
/**
* A wrapper around {@link Bootstrap} just so the process will look nicely on things like jps.
* This class starts elasticsearch.
*/
public class Elasticsearch extends Bootstrap {
public final class Elasticsearch {
public static void main(String[] args) throws Throwable {
Bootstrap.main(args);
/** no instantiation */
private Elasticsearch() {}
/**
* Main entry point for starting elasticsearch
*/
public static void main(String[] args) throws StartupError {
try {
Bootstrap.init(args);
} catch (Throwable t) {
// format exceptions to the console in a special way
// to avoid 2MB stacktraces from guice, etc.
throw new StartupError(t);
}
}
}

2
core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java
View File

@ -59,7 +59,7 @@ final class JNACLibrary {
public long rlim_max = 0;
@Override
protected List getFieldOrder() {
protected List<String> getFieldOrder() {
return Arrays.asList(new String[] { "rlim_cur", "rlim_max" });
}
}

4
core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java
View File

@ -35,7 +35,7 @@ import java.util.List;
/**
* Library for Windows/Kernel32
*/
class JNAKernel32Library {
final class JNAKernel32Library {
private static final ESLogger logger = Loggers.getLogger(JNAKernel32Library.class);
@ -148,7 +148,7 @@ class JNAKernel32Library {
public NativeLong Type;
@Override
protected List getFieldOrder() {
protected List<String> getFieldOrder() {
return Arrays.asList(new String[]{"BaseAddress", "AllocationBase", "AllocationProtect", "RegionSize", "State", "Protect", "Type"});
}
}

5
core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java
View File

@ -34,10 +34,13 @@ import static org.elasticsearch.bootstrap.JNAKernel32Library.SizeT;
*/
class JNANatives {
/** no instantiation */
private JNANatives() {}
private static final ESLogger logger = Loggers.getLogger(JNANatives.class);
// Set to true, in case native mlockall call was successful
public static boolean LOCAL_MLOCKALL = false;
static boolean LOCAL_MLOCKALL = false;
static void tryMlockall() {
int errno = Integer.MIN_VALUE;

2
core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java
View File

@ -29,6 +29,8 @@ import java.util.Map;
/** Checks that the JVM is ok and won't cause index corruption */
final class JVMCheck {
/** no instantiation */
private JVMCheck() {}
/**
* URL with latest JVM recommendations

25
core/src/main/java/org/elasticsearch/bootstrap/JarHell.java
View File

@ -25,7 +25,6 @@ import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
@ -34,14 +33,34 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
/** Simple check for duplicate class files across the classpath */
/**
* Simple check for duplicate class files across the classpath.
* <p>
* This class checks for incompatibilities in the following ways:
* <ul>
* <li>Checks that class files are not duplicated across jars.</li>
* <li>Checks any {@code X-Compile-Target-JDK} value in the jar
* manifest is compatible with current JRE</li>
* <li>Checks any {@code X-Compile-Elasticsearch-Version} value in
* the jar manifest is compatible with the current ES</li>
* </ul>
*/
public class JarHell {
/** no instantiation */
private JarHell() {}
/** Simple driver class, can be used eg. from builds. Returns non-zero on jar-hell */
@SuppressForbidden(reason = "command line tool")
public static void main(String args[]) throws Exception {

5
core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java
View File

@ -54,10 +54,7 @@ class JavaVersion implements Comparable<JavaVersion> {
}
public static boolean isValid(String value) {
if (!value.matches("^0*[0-9]+(\\.[0-9]+)*$")) {
return false;
}
return true;
return value.matches("^0*[0-9]+(\\.[0-9]+)*$");
}
private final static JavaVersion CURRENT = parse(System.getProperty("java.specification.version"));

21
core/src/main/java/org/elasticsearch/bootstrap/Natives.java
View File

@ -26,27 +26,32 @@ import org.elasticsearch.common.logging.Loggers;
* The Natives class is a wrapper class that checks if the classes necessary for calling native methods are available on
* startup. If they are not available, this class will avoid calling code that loads these classes.
*/
class Natives {
final class Natives {
/** no instantiation */
private Natives() {}
private static final ESLogger logger = Loggers.getLogger(Natives.class);
// marker to determine if the JNA class files are available to the JVM
private static boolean jnaAvailable = false;
static final boolean JNA_AVAILABLE;
static {
boolean v = false;
try {
// load one of the main JNA classes to see if the classes are available. this does not ensure that all native
// libraries are available, only the ones necessary by JNA to function
Class.forName("com.sun.jna.Native");
jnaAvailable = true;
v = true;
} catch (ClassNotFoundException e) {
logger.warn("JNA not found. native methods will be disabled.", e);
} catch (UnsatisfiedLinkError e) {
logger.warn("unable to load JNA native support library, native methods will be disabled.", e);
}
JNA_AVAILABLE = v;
}
static void tryMlockall() {
if (!jnaAvailable) {
if (!JNA_AVAILABLE) {
logger.warn("cannot mlockall because JNA is not available");
return;
}
@ -54,7 +59,7 @@ class Natives {
}
static boolean definitelyRunningAsRoot() {
if (!jnaAvailable) {
if (!JNA_AVAILABLE) {
logger.warn("cannot check if running as root because JNA is not available");
return false;
}
@ -62,7 +67,7 @@ class Natives {
}
static void tryVirtualLock() {
if (!jnaAvailable) {
if (!JNA_AVAILABLE) {
logger.warn("cannot mlockall because JNA is not available");
return;
}
@ -70,7 +75,7 @@ class Natives {
}
static void addConsoleCtrlHandler(ConsoleCtrlHandler handler) {
if (!jnaAvailable) {
if (!JNA_AVAILABLE) {
logger.warn("cannot register console handler because JNA is not available");
return;
}
@ -78,7 +83,7 @@ class Natives {
}
static boolean isMemoryLocked() {
if (!jnaAvailable) {
if (!JNA_AVAILABLE) {
return false;
}
return JNANatives.LOCAL_MLOCKALL;

92
core/src/main/java/org/elasticsearch/bootstrap/Security.java
View File

@ -38,15 +38,59 @@ import java.util.Map;
import java.util.regex.Pattern;
/**
* Initializes securitymanager with necessary permissions.
* Initializes SecurityManager with necessary permissions.
* <p>
* We use a template file (the one we test with), and add additional
* permissions based on the environment (data paths, etc)
* <h1>Initialization</h1>
* The JVM is not initially started with security manager enabled,
* instead we turn it on early in the startup process. This is a tradeoff
* between security and ease of use:
* <ul>
* <li>Assigns file permissions to user-configurable paths that can
* be specified from the command-line or {@code elasticsearch.yml}.</li>
* <li>Allows for some contained usage of native code that would not
* otherwise be permitted.</li>
* </ul>
* <p>
* <h1>Permissions</h1>
* Permissions use a policy file packaged as a resource, this file is
* also used in tests. File permissions are generated dynamically and
* combined with this policy file.
* <p>
* For each configured path, we ensure it exists and is accessible before
* granting permissions, otherwise directory creation would require
* permissions to parent directories.
* <p>
* In some exceptional cases, permissions are assigned to specific jars only,
* when they are so dangerous that general code should not be granted the
* permission, but there are extenuating circumstances.
* <p>
* Groovy scripts are assigned no permissions. This does not provide adequate
* sandboxing, as these scripts still have access to ES classes, and could
* modify members, etc that would cause bad things to happen later on their
* behalf (no package protections are yet in place, this would need some
* cleanups to the scripting apis). But still it can provide some defense for users
* that enable dynamic scripting without being fully aware of the consequences.
* <p>
* <h1>Disabling Security</h1>
* SecurityManager can be disabled completely with this setting:
* <pre>
* es.security.manager.enabled = false
* </pre>
* <p>
* <h1>Debugging Security</h1>
* A good place to start when there is a problem is to turn on security debugging:
* <pre>
* JAVA_OPTS="-Djava.security.debug=access:failure" bin/elasticsearch
* </pre>
* See <a href="https://docs.oracle.com/javase/7/docs/technotes/guides/security/troubleshooting-security.html">
* Troubleshooting Security</a> for information.
*/
final class Security {
/** no instantiation */
private Security() {}
/**
* Initializes securitymanager for the environment
* Initializes SecurityManager for the environment
* Can only happen once!
*/
static void configure(Environment environment) throws Exception {
@ -118,25 +162,25 @@ final class Security {
static Permissions createPermissions(Environment environment) throws IOException {
Permissions policy = new Permissions();
// read-only dirs
addPath(policy, environment.binFile(), "read,readlink");
addPath(policy, environment.libFile(), "read,readlink");
addPath(policy, environment.pluginsFile(), "read,readlink");
addPath(policy, environment.configFile(), "read,readlink");
addPath(policy, environment.scriptsFile(), "read,readlink");
addPath(policy, "path.home", environment.binFile(), "read,readlink");
addPath(policy, "path.home", environment.libFile(), "read,readlink");
addPath(policy, "path.plugins", environment.pluginsFile(), "read,readlink");
addPath(policy, "path.conf", environment.configFile(), "read,readlink");
addPath(policy, "path.scripts", environment.scriptsFile(), "read,readlink");
// read-write dirs
addPath(policy, environment.tmpFile(), "read,readlink,write,delete");
addPath(policy, environment.logsFile(), "read,readlink,write,delete");
addPath(policy, "java.io.tmpdir", environment.tmpFile(), "read,readlink,write,delete");
addPath(policy, "path.logs", environment.logsFile(), "read,readlink,write,delete");
if (environment.sharedDataFile() != null) {
addPath(policy, environment.sharedDataFile(), "read,readlink,write,delete");
addPath(policy, "path.shared_data", environment.sharedDataFile(), "read,readlink,write,delete");
}
for (Path path : environment.dataFiles()) {
addPath(policy, path, "read,readlink,write,delete");
addPath(policy, "path.data", path, "read,readlink,write,delete");
}
for (Path path : environment.dataWithClusterFiles()) {
addPath(policy, path, "read,readlink,write,delete");
addPath(policy, "path.data", path, "read,readlink,write,delete");
}
for (Path path : environment.repoFiles()) {
addPath(policy, path, "read,readlink,write,delete");
addPath(policy, "path.repo", path, "read,readlink,write,delete");
}
if (environment.pidFile() != null) {
// we just need permission to remove the file if its elsewhere.
@ -145,10 +189,20 @@ final class Security {
return policy;
}
/** Add access to path (and all files underneath it */
static void addPath(Permissions policy, Path path, String permissions) throws IOException {
// paths may not exist yet
ensureDirectoryExists(path);
/**
* Add access to path (and all files underneath it)
* @param policy current policy to add permissions to
* @param configurationName the configuration name associated with the path (for error messages only)
* @param path the path itself
* @param permissions set of filepermissions to grant to the path
*/
static void addPath(Permissions policy, String configurationName, Path path, String permissions) {
// paths may not exist yet, this also checks accessibility
try {
ensureDirectoryExists(path);
} catch (IOException e) {
throw new IllegalStateException("Unable to access '" + configurationName + "' (" + path + ")", e);
}
// add each path twice: once for itself, again for files underneath it
policy.add(new FilePermission(path.toString(), permissions));

7
core/src/main/java/org/elasticsearch/bootstrap/StartupError.java
View File

@ -32,7 +32,7 @@ import java.io.PrintStream;
*/
//TODO: remove this when guice is removed, and exceptions are cleaned up
//this is horrible, but its what we must do
class StartupError extends RuntimeException {
final class StartupError extends RuntimeException {
/** maximum length of a stacktrace, before we truncate it */
static final int STACKTRACE_LIMIT = 30;
@ -59,10 +59,7 @@ class StartupError extends RuntimeException {
cause = getFirstGuiceCause((CreationException)cause);
}
String message = cause.getMessage();
if (message == null) {
message = "Unknown Error";
}
String message = cause.toString();
s.println(message);
if (cause != null) {

9
core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
View File

@ -35,6 +35,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
@ -132,7 +133,11 @@ public class TransportClient extends AbstractClient {
try {
ModulesBuilder modules = new ModulesBuilder();
modules.add(new Version.Module(version));
modules.add(new PluginsModule(this.settings, pluginsService));
// plugin modules must be added here, before others or we can get crazy injection errors...
for (Module pluginModule : pluginsService.nodeModules()) {
modules.add(pluginModule);
}
modules.add(new PluginsModule(pluginsService));
modules.add(new EnvironmentModule(environment));
modules.add(new SettingsModule(this.settings));
modules.add(new NetworkModule());
@ -149,6 +154,8 @@ public class TransportClient extends AbstractClient {
modules.add(new ClientTransportModule());
modules.add(new CircuitBreakerModule(this.settings));
pluginsService.processModules(modules);
Injector injector = modules.createInjector();
injector.getInstance(TransportService.class).start();
TransportClient transportClient = new TransportClient(injector);

65
core/src/main/java/org/elasticsearch/common/inject/Modules.java
View File

@ -1,65 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.inject;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import java.lang.reflect.Constructor;
/**
*
*/
public class Modules {
public static Module createModule(Class<? extends Module> moduleClass, @Nullable Settings settings) {
Constructor<? extends Module> constructor;
try {
constructor = moduleClass.getConstructor(Settings.class);
try {
return constructor.newInstance(settings);
} catch (Exception e) {
throw new ElasticsearchException("Failed to create module [" + moduleClass + "]", e);
}
} catch (NoSuchMethodException e) {
try {
constructor = moduleClass.getConstructor();
try {
return constructor.newInstance();
} catch (Exception e1) {
throw new ElasticsearchException("Failed to create module [" + moduleClass + "]", e);
}
} catch (NoSuchMethodException e1) {
throw new ElasticsearchException("No constructor for [" + moduleClass + "]");
}
}
}
public static void processModules(Iterable<Module> modules) {
for (Module module : modules) {
if (module instanceof PreProcessModule) {
for (Module module1 : modules) {
((PreProcessModule) module).processModule(module1);
}
}
}
}
}

19
core/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java
View File

@ -31,20 +31,9 @@ public class ModulesBuilder implements Iterable<Module> {
private final List<Module> modules = Lists.newArrayList();
public ModulesBuilder add(Module... modules) {
for (Module module : modules) {
add(module);
}
return this;
}
public ModulesBuilder add(Module module) {
modules.add(module);
if (module instanceof SpawnModules) {
Iterable<? extends Module> spawned = ((SpawnModules) module).spawnModules();
for (Module spawn : spawned) {
add(spawn);
}
public ModulesBuilder add(Module... newModules) {
for (Module module : newModules) {
modules.add(module);
}
return this;
}
@ -55,7 +44,6 @@ public class ModulesBuilder implements Iterable<Module> {
}
public Injector createInjector() {
Modules.processModules(modules);
Injector injector = Guice.createInjector(modules);
Injectors.cleanCaches(injector);
// in ES, we always create all instances as if they are eager singletons
@ -65,7 +53,6 @@ public class ModulesBuilder implements Iterable<Module> {
}
public Injector createChildInjector(Injector injector) {
Modules.processModules(modules);
Injector childInjector = injector.createChildInjector(modules);
Injectors.cleanCaches(childInjector);
// in ES, we always create all instances as if they are eager singletons

3
core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java
View File

@ -19,7 +19,6 @@ package org.elasticsearch.common.inject.spi;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.elasticsearch.bootstrap.Bootstrap;
import org.elasticsearch.common.inject.*;
import org.elasticsearch.common.inject.binder.AnnotatedBindingBuilder;
import org.elasticsearch.common.inject.binder.AnnotatedConstantBindingBuilder;
@ -343,7 +342,7 @@ public final class Elements {
return builder;
}
private static ESLogger logger = Loggers.getLogger(Bootstrap.class);
private static ESLogger logger = Loggers.getLogger(Elements.class);
protected Object getSource() {
Object ret;

3
core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
View File

@ -64,6 +64,7 @@ import static org.elasticsearch.discovery.zen.ping.ZenPing.PingResponse.readPing
public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implements ZenPing {
public static final String ACTION_NAME = "internal:discovery/zen/unicast";
public static final String DISCOVERY_ZEN_PING_UNICAST_HOSTS = "discovery.zen.ping.unicast.hosts";
// these limits are per-address
public static final int LIMIT_FOREIGN_PORTS_COUNT = 1;
@ -116,7 +117,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
}
this.concurrentConnects = this.settings.getAsInt("discovery.zen.ping.unicast.concurrent_connects", 10);
String[] hostArr = this.settings.getAsArray("discovery.zen.ping.unicast.hosts");
String[] hostArr = this.settings.getAsArray(DISCOVERY_ZEN_PING_UNICAST_HOSTS);
// trim the hosts
for (int i = 0; i < hostArr.length; i++) {
hostArr[i] = hostArr[i].trim();

9
core/src/main/java/org/elasticsearch/index/IndexService.java
View File

@ -56,7 +56,6 @@ import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.InternalIndicesLifecycle;
import org.elasticsearch.indices.cache.query.IndicesQueryCache;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.plugins.ShardsPluginsModule;
import java.io.Closeable;
import java.io.IOException;
@ -317,7 +316,10 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
final boolean canDeleteShardContent = IndexMetaData.isOnSharedFilesystem(indexSettings) == false ||
(primary && IndexMetaData.isOnSharedFilesystem(indexSettings));
ModulesBuilder modules = new ModulesBuilder();
modules.add(new ShardsPluginsModule(indexSettings, pluginsService));
// plugin modules must be added here, before others or we can get crazy injection errors...
for (Module pluginModule : pluginsService.shardModules(indexSettings)) {
modules.add(pluginModule);
}
modules.add(new IndexShardModule(shardId, primary, indexSettings));
modules.add(new StoreModule(injector.getInstance(IndexStore.class).shardDirectory(), lock,
new StoreCloseListener(shardId, canDeleteShardContent, new Closeable() {
@ -327,6 +329,9 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
}
}), path));
modules.add(new DeletionPolicyModule());
pluginsService.processModules(modules);
try {
shardInjector = modules.createChildInjector(injector);
} catch (CreationException e) {

10
core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
View File

@ -73,8 +73,6 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
private String minimumShouldMatch;
private String rewrite = null;
private String fuzzyRewrite = null;
private Boolean lenient;
@ -165,11 +163,6 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
return this;
}
public MatchQueryBuilder rewrite(String rewrite) {
this.rewrite = rewrite;
return this;
}
public MatchQueryBuilder fuzzyRewrite(String fuzzyRewrite) {
this.fuzzyRewrite = fuzzyRewrite;
return this;
@ -224,9 +217,6 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
if (minimumShouldMatch != null) {
builder.field("minimum_should_match", minimumShouldMatch);
}
if (rewrite != null) {
builder.field("rewrite", rewrite);
}
if (fuzzyRewrite != null) {
builder.field("fuzzy_rewrite", fuzzyRewrite);
}

35
core/src/main/java/org/elasticsearch/indices/IndicesService.java
View File

@ -20,7 +20,12 @@
package org.elasticsearch.indices;
import com.google.common.base.Function;
import com.google.common.collect.*;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.IOUtils;
@ -35,7 +40,12 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.*;
import org.elasticsearch.common.inject.CreationException;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.Injectors;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
@ -43,7 +53,12 @@ import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.ShardLock;
import org.elasticsearch.gateway.MetaDataStateFormat;
import org.elasticsearch.index.*;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexNameModule;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.LocalNodeIdModule;
import org.elasticsearch.index.aliases.IndexAliasesServiceModule;
import org.elasticsearch.index.analysis.AnalysisModule;
import org.elasticsearch.index.analysis.AnalysisService;
@ -71,13 +86,16 @@ import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.IndexStoreModule;
import org.elasticsearch.indices.analysis.IndicesAnalysisService;
import org.elasticsearch.indices.recovery.RecoverySettings;
import org.elasticsearch.plugins.IndexPluginsModule;
import org.elasticsearch.plugins.PluginsService;
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Files;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@ -306,7 +324,10 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
modules.add(new IndexNameModule(index));
modules.add(new LocalNodeIdModule(localNodeId));
modules.add(new IndexSettingsModule(index, indexSettings));
modules.add(new IndexPluginsModule(indexSettings, pluginsService));
// plugin modules must be added here, before others or we can get crazy injection errors...
for (Module pluginModule : pluginsService.indexModules(indexSettings)) {
modules.add(pluginModule);
}
modules.add(new IndexStoreModule(indexSettings));
modules.add(new AnalysisModule(indexSettings, indicesAnalysisService));
modules.add(new SimilarityModule(indexSettings));
@ -315,6 +336,8 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
modules.add(new MapperServiceModule());
modules.add(new IndexAliasesServiceModule());
modules.add(new IndexModule(indexSettings));
pluginsService.processModules(modules);
Injector indexInjector;
try {

4
core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java
View File

@ -19,7 +19,7 @@
package org.elasticsearch.monitor.process;
import org.elasticsearch.bootstrap.Bootstrap;
import org.elasticsearch.bootstrap.BootstrapInfo;
import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
@ -136,7 +136,7 @@ public class ProcessProbe {
}
public ProcessInfo processInfo() {
return new ProcessInfo(jvmInfo().pid(), Bootstrap.isMemoryLocked());
return new ProcessInfo(jvmInfo().pid(), BootstrapInfo.isMemoryLocked());
}
public ProcessStats processStats() {

10
core/src/main/java/org/elasticsearch/node/Node.java
View File

@ -35,6 +35,7 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.Lifecycle;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.lease.Releasables;
@ -159,7 +160,11 @@ public class Node implements Releasable {
ModulesBuilder modules = new ModulesBuilder();
modules.add(new Version.Module(version));
modules.add(new CircuitBreakerModule(settings));
modules.add(new PluginsModule(settings, pluginsService));
// plugin modules must be added here, before others or we can get crazy injection errors...
for (Module pluginModule : pluginsService.nodeModules()) {
modules.add(pluginModule);
}
modules.add(new PluginsModule(pluginsService));
modules.add(new SettingsModule(settings));
modules.add(new NodeModule(this));
modules.add(new NetworkModule());
@ -187,6 +192,9 @@ public class Node implements Releasable {
modules.add(new RepositoriesModule());
modules.add(new TribeModule());
pluginsService.processModules(modules);
injector = modules.createInjector();
client = injector.getInstance(Client.class);

18
core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.node.internal;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import com.google.common.collect.UnmodifiableIterator;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.common.Booleans;
@ -40,6 +41,7 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
import static org.elasticsearch.common.Strings.cleanPath;
@ -113,13 +115,21 @@ public class InternalSettingsPreparer {
}
}
if (loadFromEnv) {
boolean settingsFileFound = false;
Set<String> foundSuffixes = Sets.newHashSet();
for (String allowedSuffix : ALLOWED_SUFFIXES) {
try {
settingsBuilder.loadFromPath(environment.configFile().resolve("elasticsearch" + allowedSuffix));
} catch (SettingsException e) {
// ignore
Path path = environment.configFile().resolve("elasticsearch" + allowedSuffix);
if (Files.exists(path)) {
if (!settingsFileFound) {
settingsBuilder.loadFromPath(path);
}
settingsFileFound = true;
foundSuffixes.add(allowedSuffix);
}
}
if (foundSuffixes.size() > 1) {
throw new SettingsException("multiple settings files found with suffixes: " + Strings.collectionToDelimitedString(foundSuffixes, ","));
}
}
}

55
core/src/main/java/org/elasticsearch/plugins/IndexPluginsModule.java
View File

@ -1,55 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugins;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.PreProcessModule;
import org.elasticsearch.common.inject.SpawnModules;
import org.elasticsearch.common.settings.Settings;
/**
*
*/
public class IndexPluginsModule extends AbstractModule implements SpawnModules, PreProcessModule {
private final Settings settings;
private final PluginsService pluginsService;
public IndexPluginsModule(Settings settings, PluginsService pluginsService) {
this.settings = settings;
this.pluginsService = pluginsService;
}
@Override
public Iterable<? extends Module> spawnModules() {
return pluginsService.indexModules(settings);
}
@Override
public void processModule(Module module) {
pluginsService.processModule(module);
}
@Override
protected void configure() {
}
}

24
core/src/main/java/org/elasticsearch/plugins/PluginsModule.java
View File

@ -20,35 +20,15 @@
package org.elasticsearch.plugins;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.PreProcessModule;
import org.elasticsearch.common.inject.SpawnModules;
import org.elasticsearch.common.settings.Settings;
/**
*
*/
public class PluginsModule extends AbstractModule implements SpawnModules, PreProcessModule {
private final Settings settings;
public class PluginsModule extends AbstractModule {
private final PluginsService pluginsService;
public PluginsModule(Settings settings, PluginsService pluginsService) {
this.settings = settings;
public PluginsModule(PluginsService pluginsService) {
this.pluginsService = pluginsService;
}
@Override
public Iterable<? extends Module> spawnModules() {
return pluginsService.nodeModules();
}
@Override
public void processModule(Module module) {
pluginsService.processModule(module);
}
@Override
protected void configure() {
bind(PluginsService.class).toInstance(pluginsService);

32
core/src/main/java/org/elasticsearch/plugins/PluginsService.java
View File

@ -22,9 +22,14 @@ package org.elasticsearch.plugins;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.cluster.node.info.PluginsInfo;
import org.elasticsearch.bootstrap.Bootstrap;
import org.elasticsearch.bootstrap.JarHell;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.MapBuilder;
@ -107,7 +112,7 @@ public class PluginsService extends AbstractComponent {
List<Bundle> bundles = getPluginBundles(environment);
tupleBuilder.addAll(loadBundles(bundles));
} catch (IOException ex) {
throw new IllegalStateException(ex);
throw new IllegalStateException("Unable to initialize plugins", ex);
}
plugins = tupleBuilder.build();
@ -279,9 +284,10 @@ public class PluginsService extends AbstractComponent {
}
static List<Bundle> getPluginBundles(Environment environment) throws IOException {
ESLogger logger = Loggers.getLogger(Bootstrap.class);
ESLogger logger = Loggers.getLogger(PluginsService.class);
Path pluginsDirectory = environment.pluginsFile();
// TODO: remove this leniency, but tests bogusly rely on it
if (!isAccessibleDirectory(pluginsDirectory, logger)) {
return Collections.emptyList();
}
@ -347,6 +353,8 @@ public class PluginsService extends AbstractComponent {
for (PluginInfo pluginInfo : bundle.plugins) {
final Plugin plugin;
if (pluginInfo.isJvm()) {
// reload lucene SPI with any new services from the plugin
reloadLuceneSPI(loader);
plugin = loadPlugin(pluginInfo.getClassname(), settings, loader);
} else {
plugin = new SitePlugin(pluginInfo.getName(), pluginInfo.getDescription());
@ -358,6 +366,24 @@ public class PluginsService extends AbstractComponent {
return plugins.build();
}
/**
* Reloads all Lucene SPI implementations using the new classloader.
* This method must be called after the new classloader has been created to
* register the services for use.
*/
static void reloadLuceneSPI(ClassLoader loader) {
// do NOT change the order of these method calls!
// Codecs:
PostingsFormat.reloadPostingsFormats(loader);
DocValuesFormat.reloadDocValuesFormats(loader);
Codec.reloadCodecs(loader);
// Analysis:
CharFilterFactory.reloadCharFilters(loader);
TokenFilterFactory.reloadTokenFilters(loader);
TokenizerFactory.reloadTokenizers(loader);
}
private Plugin loadPlugin(String className, Settings settings, ClassLoader loader) {
try {
Class<? extends Plugin> pluginClass = loader.loadClass(className).asSubclass(Plugin.class);

55
core/src/main/java/org/elasticsearch/plugins/ShardsPluginsModule.java
View File

@ -1,55 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugins;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.PreProcessModule;
import org.elasticsearch.common.inject.SpawnModules;
import org.elasticsearch.common.settings.Settings;
/**
*
*/
public class ShardsPluginsModule extends AbstractModule implements SpawnModules, PreProcessModule {
private final Settings settings;
private final PluginsService pluginsService;
public ShardsPluginsModule(Settings settings, PluginsService pluginsService) {
this.settings = settings;
this.pluginsService = pluginsService;
}
@Override
public Iterable<? extends Module> spawnModules() {
return pluginsService.shardModules(settings);
}
@Override
public void processModule(Module module) {
pluginsService.processModule(module);
}
@Override
protected void configure() {
}
}

6
core/src/main/java/org/elasticsearch/repositories/RepositoryModule.java
View File

@ -20,14 +20,8 @@
package org.elasticsearch.repositories;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.Modules;
import org.elasticsearch.common.inject.SpawnModules;
import org.elasticsearch.common.settings.Settings;
import java.util.Arrays;
import java.util.Collections;
/**
* Binds repository classes for the specific repository type.
*/

2
core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
View File

@ -727,7 +727,7 @@ public class ThreadPool extends AbstractComponent {
if (queueSize == null) {
builder.field(Fields.QUEUE_SIZE, -1);
} else {
builder.field(Fields.QUEUE_SIZE, queueSize.toString());
builder.field(Fields.QUEUE_SIZE, queueSize.singles());
}
builder.endObject();
return builder;

4
core/src/test/java/org/elasticsearch/benchmark/mapping/ManyMappingsBenchmark.java
View File

@ -21,7 +21,7 @@ package org.elasticsearch.benchmark.mapping;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.bootstrap.Bootstrap;
import org.elasticsearch.bootstrap.BootstrapForTesting;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
@ -85,7 +85,7 @@ public class ManyMappingsBenchmark {
public static void main(String[] args) throws Exception {
System.setProperty("es.logger.prefix", "");
Bootstrap.initializeNatives(true, false);
BootstrapForTesting.ensureInitialized();
Settings settings = settingsBuilder()
.put("")
.put(SETTING_NUMBER_OF_SHARDS, 5)

4
core/src/test/java/org/elasticsearch/benchmark/recovery/ReplicaRecoveryBenchmark.java
View File

@ -20,7 +20,7 @@ package org.elasticsearch.benchmark.recovery;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.recovery.ShardRecoveryResponse;
import org.elasticsearch.bootstrap.Bootstrap;
import org.elasticsearch.bootstrap.BootstrapForTesting;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
@ -57,7 +57,7 @@ public class ReplicaRecoveryBenchmark {
public static void main(String[] args) throws Exception {
System.setProperty("es.logger.prefix", "");
Bootstrap.initializeNatives(true, false);
BootstrapForTesting.ensureInitialized();
Settings settings = settingsBuilder()
.put("gateway.type", "local")

5
core/src/test/java/org/elasticsearch/benchmark/search/aggregations/GlobalOrdinalsBenchmark.java
View File

@ -21,12 +21,13 @@ package org.elasticsearch.benchmark.search.aggregations;
import com.carrotsearch.hppc.IntIntHashMap;
import com.carrotsearch.hppc.ObjectHashSet;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.benchmark.search.aggregations.TermsAggregationSearchBenchmark.StatsResult;
import org.elasticsearch.bootstrap.Bootstrap;
import org.elasticsearch.bootstrap.BootstrapForTesting;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
@ -66,7 +67,7 @@ public class GlobalOrdinalsBenchmark {
public static void main(String[] args) throws Exception {
System.setProperty("es.logger.prefix", "");
Bootstrap.initializeNatives(true, false);
BootstrapForTesting.ensureInitialized();
Random random = new Random();
Settings settings = settingsBuilder()

4
core/src/test/java/org/elasticsearch/benchmark/search/aggregations/SubAggregationSearchCollectModeBenchmark.java
View File

@ -27,7 +27,7 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.bootstrap.Bootstrap;
import org.elasticsearch.bootstrap.BootstrapForTesting;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.StopWatch;
@ -71,7 +71,7 @@ public class SubAggregationSearchCollectModeBenchmark {
static Node[] nodes;
public static void main(String[] args) throws Exception {
Bootstrap.initializeNatives(true, false);
BootstrapForTesting.ensureInitialized();
Random random = new Random();
Settings settings = settingsBuilder()

5
core/src/test/java/org/elasticsearch/benchmark/search/aggregations/TermsAggregationSearchAndIndexingBenchmark.java
View File

@ -20,13 +20,14 @@ package org.elasticsearch.benchmark.search.aggregations;
import com.carrotsearch.hppc.ObjectScatterSet;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.bootstrap.Bootstrap;
import org.elasticsearch.bootstrap.BootstrapForTesting;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.settings.Settings;
@ -71,7 +72,7 @@ public class TermsAggregationSearchAndIndexingBenchmark {
static Node[] nodes;
public static void main(String[] args) throws Exception {
Bootstrap.initializeNatives(true, false);
BootstrapForTesting.ensureInitialized();
Settings settings = settingsBuilder()
.put("refresh_interval", "-1")
.put(SETTING_NUMBER_OF_SHARDS, 1)

4
core/src/test/java/org/elasticsearch/benchmark/search/aggregations/TermsAggregationSearchBenchmark.java
View File

@ -28,7 +28,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.bootstrap.Bootstrap;
import org.elasticsearch.bootstrap.BootstrapForTesting;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.StopWatch;
@ -99,7 +99,7 @@ public class TermsAggregationSearchBenchmark {
}
public static void main(String[] args) throws Exception {
Bootstrap.initializeNatives(true, false);
BootstrapForTesting.ensureInitialized();
Random random = new Random();
Settings settings = settingsBuilder()

2
core/src/test/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
View File

@ -101,7 +101,7 @@ public class BootstrapForTesting {
}
}
// java.io.tmpdir
Security.addPath(perms, javaTmpDir, "read,readlink,write,delete");
Security.addPath(perms, "java.io.tmpdir", javaTmpDir, "read,readlink,write,delete");
// custom test config file
if (Strings.hasLength(System.getProperty("tests.config"))) {
perms.add(new FilePermission(System.getProperty("tests.config"), "read,readlink"));

2
core/src/test/java/org/elasticsearch/bootstrap/SecurityTests.java
View File

@ -244,7 +244,7 @@ public class SecurityTests extends ESTestCase {
assumeNoException("test cannot create symbolic links with security manager enabled", e);
}
Permissions permissions = new Permissions();
Security.addPath(permissions, link, "read");
Security.addPath(permissions, "testing", link, "read");
assertExactPermissions(new FilePermission(link.toString(), "read"), permissions);
assertExactPermissions(new FilePermission(link.resolve("foo").toString(), "read"), permissions);
assertExactPermissions(new FilePermission(target.toString(), "read"), permissions);

32
core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
View File

@ -1001,38 +1001,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
return list.get(0);
}
private void ensureStableCluster(int nodeCount) {
ensureStableCluster(nodeCount, TimeValue.timeValueSeconds(30));
}
private void ensureStableCluster(int nodeCount, TimeValue timeValue) {
ensureStableCluster(nodeCount, timeValue, false, null);
}
private void ensureStableCluster(int nodeCount, @Nullable String viaNode) {
ensureStableCluster(nodeCount, TimeValue.timeValueSeconds(30), false, viaNode);
}
private void ensureStableCluster(int nodeCount, TimeValue timeValue, boolean local, @Nullable String viaNode) {
if (viaNode == null) {
viaNode = randomFrom(internalCluster().getNodeNames());
}
logger.debug("ensuring cluster is stable with [{}] nodes. access node: [{}]. timeout: [{}]", nodeCount, viaNode, timeValue);
ClusterHealthResponse clusterHealthResponse = client(viaNode).admin().cluster().prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForNodes(Integer.toString(nodeCount))
.setTimeout(timeValue)
.setLocal(local)
.setWaitForRelocatingShards(0)
.get();
if (clusterHealthResponse.isTimedOut()) {
ClusterStateResponse stateResponse = client(viaNode).admin().cluster().prepareState().get();
fail("failed to reach a stable cluster of [" + nodeCount + "] nodes. Tried via [" + viaNode + "]. last cluster state:\n"
+ stateResponse.getState().prettyPrint());
}
assertThat(clusterHealthResponse.isTimedOut(), is(false));
}
private ClusterState getNodeClusterState(String node) {
return client(node).admin().cluster().prepareState().setLocal(true).get().getState();
}

8
core/src/test/java/org/elasticsearch/discovery/ZenUnicastDiscoveryIT.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.discovery;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
@ -81,12 +82,15 @@ public class ZenUnicastDiscoveryIT extends ESIntegTestCase {
int currentNumNodes = randomIntBetween(3, 5);
final int min_master_nodes = currentNumNodes / 2 + 1;
int currentNumOfUnicastHosts = randomIntBetween(min_master_nodes, currentNumNodes);
final Settings settings = Settings.settingsBuilder().put("discovery.zen.minimum_master_nodes", min_master_nodes).build();
final Settings settings = Settings.settingsBuilder()
.put("discovery.zen.join_timeout", TimeValue.timeValueSeconds(10))
.put("discovery.zen.minimum_master_nodes", min_master_nodes)
.build();
discoveryConfig = new ClusterDiscoveryConfiguration.UnicastZen(currentNumNodes, currentNumOfUnicastHosts, settings);
List<String> nodes = internalCluster().startNodesAsync(currentNumNodes).get();
ensureGreen();
ensureStableCluster(currentNumNodes);
DiscoveryNode masterDiscoNode = null;
for (String node : nodes) {

4
core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java
View File

@ -20,7 +20,7 @@
package org.elasticsearch.monitor.process;
import org.apache.lucene.util.Constants;
import org.elasticsearch.bootstrap.Bootstrap;
import org.elasticsearch.bootstrap.BootstrapInfo;
import org.elasticsearch.test.ESTestCase;
import org.junit.Test;
@ -37,7 +37,7 @@ public class ProcessProbeTests extends ESTestCase {
assertNotNull(info);
assertThat(info.getRefreshInterval(), greaterThanOrEqualTo(0L));
assertThat(info.getId(), equalTo(jvmInfo().pid()));
assertThat(info.isMlockall(), equalTo(Bootstrap.isMemoryLocked()));
assertThat(info.isMlockall(), equalTo(BootstrapInfo.isMemoryLocked()));
}
@Test

64
core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
View File

@ -23,15 +23,17 @@ import org.elasticsearch.common.cli.CliToolTestCase;
import org.elasticsearch.common.cli.Terminal;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.env.Environment;
import org.elasticsearch.test.ESTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@ -42,6 +44,8 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.hamcrest.Matchers.*;
public class InternalSettingsPreparerTests extends ESTestCase {
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Before
public void setupSystemProperties() {
@ -75,29 +79,6 @@ public class InternalSettingsPreparerTests extends ESTestCase {
assertThat(tuple.v1().get("node.zone"), equalTo("bar"));
}
@Test
public void testAlternateConfigFileSuffixes() throws Exception {
InputStream yaml = getClass().getResourceAsStream("/config/elasticsearch.yaml");
InputStream json = getClass().getResourceAsStream("/config/elasticsearch.json");
InputStream properties = getClass().getResourceAsStream("/config/elasticsearch.properties");
Path home = createTempDir();
Path config = home.resolve("config");
Files.createDirectory(config);
Files.copy(yaml, config.resolve("elasticsearch.yaml"));
Files.copy(json, config.resolve("elasticsearch.json"));
Files.copy(properties, config.resolve("elasticsearch.properties"));
// test that we can read config files with .yaml, .json, and .properties suffixes
Tuple<Settings, Environment> tuple = InternalSettingsPreparer.prepareSettings(settingsBuilder()
.put("config.ignore_system_properties", true)
.put("path.home", home)
.build(), true);
assertThat(tuple.v1().get("yaml.config.exists"), equalTo("true"));
assertThat(tuple.v1().get("json.config.exists"), equalTo("true"));
assertThat(tuple.v1().get("properties.config.exists"), equalTo("true"));
}
@Test
public void testReplacePromptPlaceholders() {
final List<String> replacedSecretProperties = new ArrayList<>();
@ -235,4 +216,37 @@ public class InternalSettingsPreparerTests extends ESTestCase {
assertThat(settings.get("name"), is("prompted name 0"));
assertThat(settings.get("node.name"), is("prompted name 0"));
}
@Test(expected = SettingsException.class)
public void testGarbageIsNotSwallowed() throws IOException {
InputStream garbage = getClass().getResourceAsStream("/config/garbage/garbage.yml");
Path home = createTempDir();
Path config = home.resolve("config");
Files.createDirectory(config);
Files.copy(garbage, config.resolve("elasticsearch.yml"));
InternalSettingsPreparer.prepareSettings(settingsBuilder()
.put("config.ignore_system_properties", true)
.put("path.home", home)
.build(), true);
}
public void testMultipleSettingsFileNotAllowed() throws IOException {
InputStream yaml = getClass().getResourceAsStream("/config/elasticsearch.yaml");
InputStream properties = getClass().getResourceAsStream("/config/elasticsearch.properties");
Path home = createTempDir();
Path config = home.resolve("config");
Files.createDirectory(config);
Files.copy(yaml, config.resolve("elasticsearch.yaml"));
Files.copy(properties, config.resolve("elasticsearch.properties"));
expectedException.expect(SettingsException.class);
expectedException.expectMessage("multiple settings files found with suffixes: ");
expectedException.expectMessage("yaml");
expectedException.expectMessage("properties");
InternalSettingsPreparer.prepareSettings(settingsBuilder()
.put("config.ignore_system_properties", true)
.put("path.home", home)
.build(), true);
}
}

33
core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java
View File

@ -29,6 +29,7 @@ import com.google.common.base.Predicate;
import com.google.common.collect.Lists;
import org.apache.http.impl.client.HttpClients;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
@ -1088,6 +1089,38 @@ public abstract class ESIntegTestCase extends ESTestCase {
return ensureGreen(indices);
}
protected void ensureStableCluster(int nodeCount) {
ensureStableCluster(nodeCount, TimeValue.timeValueSeconds(30));
}
protected void ensureStableCluster(int nodeCount, TimeValue timeValue) {
ensureStableCluster(nodeCount, timeValue, false, null);
}
protected void ensureStableCluster(int nodeCount, @Nullable String viaNode) {
ensureStableCluster(nodeCount, TimeValue.timeValueSeconds(30), false, viaNode);
}
protected void ensureStableCluster(int nodeCount, TimeValue timeValue, boolean local, @Nullable String viaNode) {
if (viaNode == null) {
viaNode = randomFrom(internalCluster().getNodeNames());
}
logger.debug("ensuring cluster is stable with [{}] nodes. access node: [{}]. timeout: [{}]", nodeCount, viaNode, timeValue);
ClusterHealthResponse clusterHealthResponse = client(viaNode).admin().cluster().prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForNodes(Integer.toString(nodeCount))
.setTimeout(timeValue)
.setLocal(local)
.setWaitForRelocatingShards(0)
.get();
if (clusterHealthResponse.isTimedOut()) {
ClusterStateResponse stateResponse = client(viaNode).admin().cluster().prepareState().get();
fail("failed to reach a stable cluster of [" + nodeCount + "] nodes. Tried via [" + viaNode + "]. last cluster state:\n"
+ stateResponse.getState().prettyPrint());
}
assertThat(clusterHealthResponse.isTimedOut(), is(false));
}
/**
* Syntactic sugar for:
* <pre>

5
core/src/test/java/org/elasticsearch/test/InternalTestCluster.java
View File

@ -227,11 +227,6 @@ public final class InternalTestCluster extends TestCluster {
private ServiceDisruptionScheme activeDisruptionScheme;
private String nodeMode;
public InternalTestCluster(String nodeMode, long clusterSeed, Path baseDir, int minNumDataNodes, int maxNumDataNodes, String clusterName, int numClientNodes,
boolean enableHttpPipelining, String nodePrefix) {
this(nodeMode, clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, DEFAULT_SETTINGS_SOURCE, numClientNodes, enableHttpPipelining, nodePrefix);
}
public InternalTestCluster(String nodeMode, long clusterSeed, Path baseDir,
int minNumDataNodes, int maxNumDataNodes, String clusterName, SettingsSource settingsSource, int numClientNodes,
boolean enableHttpPipelining, String nodePrefix) {

19
core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
View File

@ -99,4 +99,23 @@ public class ThreadPoolSerializationTests extends ESTestCase {
assertThat(threadPool.info("index").getQueueSize(), is(nullValue()));
terminate(threadPool);
}
@Test
public void testThatToXContentWritesInteger() throws Exception {
ThreadPool.Info info = new ThreadPool.Info("foo", "search", 1, 10, TimeValue.timeValueMillis(3000), SizeValue.parseSizeValue("1k"));
XContentBuilder builder = jsonBuilder();
builder.startObject();
info.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
BytesReference bytesReference = builder.bytes();
Map<String, Object> map;
try (XContentParser parser = XContentFactory.xContent(bytesReference).createParser(bytesReference)) {
map = parser.map();
}
assertThat(map, hasKey("foo"));
map = (Map<String, Object>) map.get("foo");
assertThat(map, hasKey("queue_size"));
assertThat(map.get("queue_size").toString(), is("1000"));
}
}

15
core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
View File

@ -42,7 +42,6 @@ import org.elasticsearch.client.FilterClient;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.PreProcessModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.http.HttpServerTransport;
@ -401,22 +400,18 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
public Collection<Module> nodeModules() {
return Collections.<Module>singletonList(new ActionLoggingModule());
}
public void onModule(ActionModule module) {
module.registerFilter(LoggingFilter.class);
}
}
public static class ActionLoggingModule extends AbstractModule implements PreProcessModule {
public static class ActionLoggingModule extends AbstractModule {
@Override
protected void configure() {
bind(LoggingFilter.class).asEagerSingleton();
}
@Override
public void processModule(Module module) {
if (module instanceof ActionModule) {
((ActionModule)module).registerFilter(LoggingFilter.class);
}
}
}
public static class LoggingFilter extends ActionFilter.Simple {

36
core/src/test/java/org/elasticsearch/tribe/TribeIT.java
View File

@ -23,6 +23,8 @@ import com.google.common.collect.ImmutableMap;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.ClusterState;
@ -32,11 +34,14 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.discovery.MasterNotDiscoveredException;
import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.InternalTestCluster;
import org.elasticsearch.test.SettingsSource;
import org.elasticsearch.test.TestCluster;
import org.junit.After;
import org.junit.AfterClass;
@ -44,6 +49,8 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@ -69,8 +76,21 @@ public class TribeIT extends ESIntegTestCase {
@BeforeClass
public static void setupSecondCluster() throws Exception {
ESIntegTestCase.beforeClass();
SettingsSource source = new SettingsSource() {
@Override
public Settings node(int nodeOrdinal) {
final int base = InternalTestCluster.BASE_PORT + 1000;
return Settings.builder().put("transport.tcp.port", base + "-" + (base + 100)).build();
}
@Override
public Settings transportClient() {
return node(0);
}
};
// create another cluster
cluster2 = new InternalTestCluster(InternalTestCluster.configuredNodeMode(), randomLong(), createTempDir(), 2, 2, Strings.randomBase64UUID(getRandom()), 0, false, SECOND_CLUSTER_NODE_PREFIX);
cluster2 = new InternalTestCluster(InternalTestCluster.configuredNodeMode(), randomLong(), createTempDir(), 2, 2, Strings.randomBase64UUID(getRandom()), source, 0, false, SECOND_CLUSTER_NODE_PREFIX);
cluster2.beforeTest(getRandom(), 0.1);
cluster2.ensureAtLeastNumDataNodes(2);
}
@ -109,6 +129,10 @@ public class TribeIT extends ESIntegTestCase {
tribe1Defaults.put("tribe.t1." + entry.getKey(), entry.getValue());
tribe2Defaults.put("tribe.t2." + entry.getKey(), entry.getValue());
}
// give each tribe it's unicast hosts to connect to
tribe1Defaults.putArray("tribe.t1." + UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS, getUnicastHosts(internalCluster().client()));
tribe1Defaults.putArray("tribe.t2." + UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS, getUnicastHosts(cluster2.client()));
Settings merged = Settings.builder()
.put("tribe.t1.cluster.name", internalCluster().getClusterName())
.put("tribe.t2.cluster.name", cluster2.getClusterName())
@ -421,4 +445,14 @@ public class TribeIT extends ESIntegTestCase {
}
return count;
}
public String[] getUnicastHosts(Client client) {
ArrayList<String> unicastHosts = new ArrayList<>();
NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().clear().setTransport(true).get();
for (NodeInfo info : nodeInfos.getNodes()) {
TransportAddress address = info.getTransport().getAddress().publishAddress();
unicastHosts.add(address.getAddress() + ":" + address.getPort());
}
return unicastHosts.toArray(new String[unicastHosts.size()]);
}
}

7
core/src/test/resources/config/garbage/garbage.yml Normal file
View File

@ -0,0 +1,7 @@
SKDFLK@$#L%@KL#%L#@$#@L$ #L$@$ #L@K$#L $L $K#L#@L $#L
!!@!@$(#%#)(@)% #(%)
#(%#@)%@#)% (@#%()
()#%@#% (@ )%@%(@#)% @( %)@ %(@)
)(%)@()(%)()(#%)@#
node.name: "Hiro Takachiho"

8
dev-tools/prepare_release_candidate.py
View File

@ -86,7 +86,7 @@ enabled=1
def run(command, env_vars=None):
if env_vars:
for key, value in env_vars.iter_items():
for key, value in env_vars.items():
os.putenv(key, value)
if os.system('%s' % (command)):
raise RuntimeError(' FAILED: %s' % (command))
@ -223,10 +223,10 @@ if __name__ == "__main__":
shutil.rmtree(localRepoElasticsearch)
if install_only:
mvn_targets = 'install'
mvn_target = 'install'
else:
mvn_targets = 'install deploy'
install_command = 'mvn clean %s -Prelease -Dskip.integ.tests=true -Dgpg.keyname="%s" -Dpackaging.rpm.rpmbuild=/usr/bin/rpmbuild -Drpm.sign=true -Dmaven.repo.local=%s -Dno.commit.pattern="\\bno(n|)commit\\b" -Dforbidden.test.signatures=""' % (mvn_targets, gpg_key, localRepo)
mvn_target = 'deploy'
install_command = 'mvn clean %s -Prelease -Dskip.integ.tests=true -Dgpg.keyname="%s" -Dpackaging.rpm.rpmbuild=/usr/bin/rpmbuild -Drpm.sign=true -Dmaven.repo.local=%s -Dno.commit.pattern="\\bno(n|)commit\\b" -Dforbidden.test.signatures=""' % (mvn_target, gpg_key, localRepo)
clean_repo_command = 'find %s -name _remote.repositories -exec rm {} \;' % (localRepoElasticsearch)
rename_metadata_files_command = 'for i in $(find %s -name "maven-metadata-local.xml*") ; do mv "$i" "${i/-local/}" ; done' % (localRepoElasticsearch)
s3_sync_command = 's3cmd sync %s s3://download.elasticsearch.org/elasticsearch/staging/%s-%s/org/' % (localRepoElasticsearch, release_version, shortHash)

12
docs/reference/aggregations/pipeline/derivative-aggregation.asciidoc
View File

@ -54,7 +54,7 @@ The following snippet calculates the derivative of the total monthly `sales`:
},
"sales_deriv": {
"derivative": {
"buckets_paths": "sales" <1>
"buckets_path": "sales" <1>
}
}
}
@ -63,7 +63,7 @@ The following snippet calculates the derivative of the total monthly `sales`:
}
--------------------------------------------------
<1> `bucket_paths` instructs this derivative aggregation to use the output of the `sales` aggregation for the derivative
<1> `buckets_path` instructs this derivative aggregation to use the output of the `sales` aggregation for the derivative
And the following may be the response:
@ -137,12 +137,12 @@ monthly sales:
},
"sales_deriv": {
"derivative": {
"buckets_paths": "sales"
"buckets_path": "sales"
}
},
"sales_2nd_deriv": {
"derivative": {
"buckets_paths": "sales_deriv" <1>
"buckets_path": "sales_deriv" <1>
}
}
}
@ -151,7 +151,7 @@ monthly sales:
}
--------------------------------------------------
<1> `bucket_paths` for the second derivative points to the name of the first derivative
<1> `buckets_path` for the second derivative points to the name of the first derivative
And the following may be the response:
@ -225,7 +225,7 @@ of the total sales per month but ask for the derivative of the sales as in the u
},
"sales_deriv": {
"derivative": {
"buckets_paths": "sales",
"buckets_path": "sales",
"unit": "day" <1>
}
}

4
docs/reference/migration/migrate_2_0/stats.asciidoc
View File

@ -55,3 +55,7 @@ headers by default. Verbosity can be turned off with the `v` parameter:
GET _cat/shards?v=0
-----------------
==== Nodes Stats API
Queue lengths are now reported as basic numeric so they can easily processed by code. Before we used a human
readable format. For example, a queue with 1,000 items is now reported as `1000` instead of `1k`.

2
docs/reference/search/suggesters/completion-suggest.asciidoc
View File

@ -165,7 +165,7 @@ curl -X POST 'localhost:9200/music/_suggest?pretty' -d '{
"song-suggest" : [ {
"text" : "n",
"offset" : 0,
"length" : 4,
"length" : 1,
"options" : [ {
"text" : "Nirvana - Nevermind",
"score" : 34.0, "payload" : {"artistId":2321}