honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into construct_it_yourself

This commit is contained in:
Ryan Ernst 2015-08-18 14:15:08 -07:00
parent c3a22e6f0e 0ffd99cca3
commit 7393068417
51 changed files with 428 additions and 544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/pom.xml
View File

@ -161,10 +161,6 @@
<groupId>org.hdrhistogram</groupId>
<artifactId>HdrHistogram</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>

1
core/src/main/assemblies/common-bin.xml
View File

@ -23,7 +23,6 @@
<include>com.ning:compress-lzf</include>
<include>com.github.spullara.mustache.java:compiler</include>
<include>com.tdunning:t-digest</include>
<include>org.apache.commons:commons-lang3</include>
<include>commons-cli:commons-cli</include>
<include>com.twitter:jsr166e</include>
<include>org.hdrhistogram:HdrHistogram</include>

2
core/src/main/java/org/elasticsearch/common/cli/CheckFileCommand.java
View File

@ -76,7 +76,7 @@ public abstract class CheckFileCommand extends CliTool.Command {
if (paths != null && paths.length > 0) {
for (Path path : paths) {
try {
boolean supportsPosixPermissions = Files.getFileStore(path).supportsFileAttributeView(PosixFileAttributeView.class);
boolean supportsPosixPermissions = Environment.getFileStore(path).supportsFileAttributeView(PosixFileAttributeView.class);
if (supportsPosixPermissions) {
PosixFileAttributes attributes = Files.readAttributes(path, PosixFileAttributes.class);
permissions.put(path, attributes.permissions());

1
core/src/main/java/org/elasticsearch/common/cli/CliTool.java
View File

@ -23,7 +23,6 @@ import com.google.common.base.Preconditions;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.GnuParser;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;

72
core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java
View File

@ -22,9 +22,9 @@ package org.elasticsearch.common.collect;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import com.google.common.collect.UnmodifiableIterator;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.lucene.util.mutable.MutableValueInt;
import java.lang.reflect.Array;
import java.util.*;
/**
@ -134,12 +134,13 @@ public final class CopyOnWriteHashMap<K, V> extends AbstractMap<K, V> {
@Override
V get(Object key, int hash) {
final int slot = ArrayUtils.indexOf(keys, key);
if (slot < 0) {
return null;
} else {
return values[slot];
for (int i = 0; i < keys.length; i++) {
if (key.equals(keys[i])) {
return values[i];
}
}
return null;
}
private static <T> T[] replace(T[] array, int index, T value) {
@ -151,14 +152,20 @@ public final class CopyOnWriteHashMap<K, V> extends AbstractMap<K, V> {
@Override
Leaf<K, V> put(K key, int hash, int hashBits, V value, MutableValueInt newValue) {
assert hashBits <= 0 : hashBits;
final int slot = ArrayUtils.indexOf(keys, key);
int slot = -1;
for (int i = 0; i < keys.length; i++) {
if (key.equals(keys[i])) {
slot = i;
break;
}
}
final K[] keys2;
final V[] values2;
if (slot < 0) {
keys2 = ArrayUtils.add(keys, key);
values2 = ArrayUtils.add(values, value);
keys2 = appendElement(keys, key);
values2 = appendElement(values, value);
newValue.value = 1;
} else {
keys2 = replace(keys, slot, key);
@ -170,16 +177,49 @@ public final class CopyOnWriteHashMap<K, V> extends AbstractMap<K, V> {
@Override
Leaf<K, V> remove(Object key, int hash) {
final int slot = ArrayUtils.indexOf(keys, key);
int slot = -1;
for (int i = 0; i < keys.length; i++) {
if (key.equals(keys[i])) {
slot = i;
break;
}
}
if (slot < 0) {
return this;
}
final K[] keys2 = ArrayUtils.remove(keys, slot);
final V[] values2 = ArrayUtils.remove(values, slot);
final K[] keys2 = removeArrayElement(keys, slot);
final V[] values2 = removeArrayElement(values, slot);
return new Leaf<>(keys2, values2);
}
}
private static <T> T[] removeArrayElement(T[] array, int index) {
final Object result = Array.newInstance(array.getClass().getComponentType(), array.length - 1);
System.arraycopy(array, 0, result, 0, index);
if (index < array.length - 1) {
System.arraycopy(array, index + 1, result, index, array.length - index - 1);
}
return (T[]) result;
}
public static <T> T[] appendElement(final T[] array, final T element) {
final T[] newArray = Arrays.copyOf(array, array.length + 1);
newArray[newArray.length - 1] = element;
return newArray;
}
public static <T> T[] insertElement(final T[] array, final T element, final int index) {
final T[] result = Arrays.copyOf(array, array.length + 1);
System.arraycopy(array, 0, result, 0, index);
result[index] = element;
if (index < array.length) {
System.arraycopy(array, index, result, index + 1, array.length - index);
}
return result;
}
/**
* An inner node in this trie. Inner nodes store up to 64 key-value pairs
* and use a bitmap in order to associate hashes to them. For example, if
@ -320,8 +360,8 @@ public final class CopyOnWriteHashMap<K, V> extends AbstractMap<K, V> {
private InnerNode<K, V> putNew(K key, int hash6, int slot, V value) {
final long mask2 = mask | (1L << hash6);
final K[] keys2 = ArrayUtils.add(keys, slot, key);
final Object[] subNodes2 = ArrayUtils.add(subNodes, slot, value);
final K[] keys2 = insertElement(keys, key, slot);
final Object[] subNodes2 = insertElement(subNodes, value, slot);
return new InnerNode<>(mask2, keys2, subNodes2);
}
@ -342,8 +382,8 @@ public final class CopyOnWriteHashMap<K, V> extends AbstractMap<K, V> {
private InnerNode<K, V> removeSlot(int hash6, int slot) {
final long mask2 = mask & ~(1L << hash6);
final K[] keys2 = ArrayUtils.remove(keys, slot);
final Object[] subNodes2 = ArrayUtils.remove(subNodes, slot);
final K[] keys2 = removeArrayElement(keys, slot);
final Object[] subNodes2 = removeArrayElement(subNodes, slot);
return new InnerNode<>(mask2, keys2, subNodes2);
}

11
core/src/main/java/org/elasticsearch/common/geo/builders/BasePolygonBuilder.java
View File

@ -23,7 +23,7 @@ import com.google.common.collect.Sets;
import com.spatial4j.core.exception.InvalidShapeException;
import com.spatial4j.core.shape.Shape;
import com.vividsolutions.jts.geom.*;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
@ -98,7 +98,6 @@ public abstract class BasePolygonBuilder<E extends BasePolygonBuilder<E>> extend
/**
* build new hole to the polygon
* @param hole linear ring defining the hole
* @return this
*/
public Ring<E> hole() {
@ -285,7 +284,7 @@ public abstract class BasePolygonBuilder<E extends BasePolygonBuilder<E>> extend
Edge current = edge;
Edge prev = edge;
// bookkeep the source and sink of each visited coordinate
HashMap<Coordinate, Pair<Edge, Edge>> visitedEdge = new HashMap<>();
HashMap<Coordinate, Tuple<Edge, Edge>> visitedEdge = new HashMap<>();
do {
current.coordinate = shift(current.coordinate, shiftOffset);
current.component = id;
@ -301,7 +300,7 @@ public abstract class BasePolygonBuilder<E extends BasePolygonBuilder<E>> extend
// since we're splitting connected components, we want the edges method to visit
// the newly separated component
final int visitID = -id;
Edge firstAppearance = visitedEdge.get(current.coordinate).getRight();
Edge firstAppearance = visitedEdge.get(current.coordinate).v2();
// correct the graph pointers by correcting the 'next' pointer for both the
// first appearance and this appearance of the edge
Edge temp = firstAppearance.next;
@ -312,12 +311,12 @@ public abstract class BasePolygonBuilder<E extends BasePolygonBuilder<E>> extend
// a non-visited value (anything positive)
do {
prev.component = visitID;
prev = visitedEdge.get(prev.coordinate).getLeft();
prev = visitedEdge.get(prev.coordinate).v1();
++splitIndex;
} while (!current.coordinate.equals(prev.coordinate));
++connectedComponents;
} else {
visitedEdge.put(current.coordinate, Pair.of(prev, current));
visitedEdge.put(current.coordinate, new Tuple<Edge, Edge>(prev, current));
}
edges.add(current);
prev = current;

11
core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
View File

@ -26,9 +26,8 @@ import com.spatial4j.core.shape.jts.JtsGeometry;
import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.Geometry;
import com.vividsolutions.jts.geom.GeometryFactory;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.unit.DistanceUnit.Distance;
@ -487,7 +486,7 @@ public abstract class ShapeBuilder implements ToXContent {
return top;
}
private static final Pair range(Coordinate[] points, int offset, int length) {
private static final double[] range(Coordinate[] points, int offset, int length) {
double minX = points[0].x;
double maxX = points[0].x;
double minY = points[0].y;
@ -507,7 +506,7 @@ public abstract class ShapeBuilder implements ToXContent {
maxY = points[offset + i].y;
}
}
return Pair.of(Pair.of(minX, maxX), Pair.of(minY, maxY));
return new double[] {minX, maxX, minY, maxY};
}
/**
@ -585,8 +584,8 @@ public abstract class ShapeBuilder implements ToXContent {
// and convert to a right handed system
// compute the bounding box and calculate range
Pair<Pair, Pair> range = range(points, offset, length);
final double rng = (Double)range.getLeft().getRight() - (Double)range.getLeft().getLeft();
double[] range = range(points, offset, length);
final double rng = range[1] - range[0];
// translate the points if the following is true
// 1. shell orientation is cw and range is greater than a hemisphere (180 degrees) but not spanning 2 hemispheres
// (translation would result in a collapsed poly)

7
core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java
View File

@ -131,13 +131,16 @@ public abstract class ExtensionPoint {
* the settings object.
*
* @param binder the binder to use
* @param settings the settings to look up the key to find the implemetation to bind
* @param settings the settings to look up the key to find the implementation to bind
* @param settingsKey the key to use with the settings
* @param defaultValue the default value if they settings doesn't contain the key
* @param defaultValue the default value if the settings do not contain the key, or null if there is no default
* @return the actual bound type key
*/
public String bindType(Binder binder, Settings settings, String settingsKey, String defaultValue) {
final String type = settings.get(settingsKey, defaultValue);
if (type == null) {
throw new IllegalArgumentException("Missing setting [" + settingsKey + "]");
}
final Class<? extends T> instance = getExtension(type);
if (instance == null) {
throw new IllegalArgumentException("Unknown [" + this.name + "] type [" + type + "]");

66
core/src/main/java/org/elasticsearch/env/ESFileStore.java vendored
View File

@ -26,10 +26,12 @@ import org.elasticsearch.common.io.PathUtils;
import java.io.IOException;
import java.nio.file.FileStore;
import java.nio.file.FileSystemException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileAttributeView;
import java.nio.file.attribute.FileStoreAttributeView;
import java.util.Arrays;
/**
* Implementation of FileStore that supports
@ -73,13 +75,16 @@ class ESFileStore extends FileStore {
}
}
/** Files.getFileStore(Path) useless here! Don't complain, just try it yourself. */
static FileStore getMatchingFileStore(Path path, FileStore fileStores[]) throws IOException {
FileStore store = Files.getFileStore(path);
/**
* Files.getFileStore(Path) useless here! Don't complain, just try it yourself.
*/
@SuppressForbidden(reason = "works around the bugs")
static FileStore getMatchingFileStore(Path path, FileStore fileStores[]) throws IOException {
if (Constants.WINDOWS) {
return store; // be defensive, don't even try to do anything fancy.
return getFileStoreWindows(path, fileStores);
}
FileStore store = Files.getFileStore(path);
try {
String mount = getMountPointLinux(store);
@ -110,6 +115,57 @@ class ESFileStore extends FileStore {
// fall back to crappy one we got from Files.getFileStore
return store;
}
/**
* remove this code and just use getFileStore for windows on java 9
* works around https://bugs.openjdk.java.net/browse/JDK-8034057
*/
@SuppressForbidden(reason = "works around https://bugs.openjdk.java.net/browse/JDK-8034057")
static FileStore getFileStoreWindows(Path path, FileStore fileStores[]) throws IOException {
assert Constants.WINDOWS;
try {
return Files.getFileStore(path);
} catch (FileSystemException possibleBug) {
final char driveLetter;
// look for a drive letter to see if its the SUBST bug,
// it might be some other type of path, like a windows share
// if something goes wrong, we just deliver the original exception
try {
String root = path.toRealPath().getRoot().toString();
if (root.length() < 2) {
throw new RuntimeException("root isn't a drive letter: " + root);
}
driveLetter = Character.toLowerCase(root.charAt(0));
if (Character.isAlphabetic(driveLetter) == false || root.charAt(1) != ':') {
throw new RuntimeException("root isn't a drive letter: " + root);
}
} catch (Throwable checkFailed) {
// something went wrong,
possibleBug.addSuppressed(checkFailed);
throw possibleBug;
}
// we have a drive letter: the hack begins!!!!!!!!
try {
// we have no choice but to parse toString of all stores and find the matching drive letter
for (FileStore store : fileStores) {
String toString = store.toString();
int length = toString.length();
if (length > 3 && toString.endsWith(":)") && toString.charAt(length - 4) == '(') {
if (Character.toLowerCase(toString.charAt(length - 3)) == driveLetter) {
return store;
}
}
}
throw new RuntimeException("no filestores matched");
} catch (Throwable weTried) {
IOException newException = new IOException("Unable to retrieve filestore for '" + path + "', tried matching against " + Arrays.toString(fileStores), weTried);
newException.addSuppressed(possibleBug);
throw newException;
}
}
}
@Override
public String name() {

31
core/src/main/java/org/elasticsearch/env/Environment.java vendored
View File

@ -19,6 +19,7 @@
package org.elasticsearch.env;
import org.apache.lucene.util.Constants;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.io.PathUtils;
@ -302,9 +303,37 @@ public class Environment {
* <li>Only requires the security permissions of {@link Files#getFileStore(Path)},
* no permissions to the actual mount point are required.
* <li>Exception handling has the same semantics as {@link Files#getFileStore(Path)}.
* <li>Works around https://bugs.openjdk.java.net/browse/JDK-8034057.
* </ul>
*/
public FileStore getFileStore(Path path) throws IOException {
public static FileStore getFileStore(Path path) throws IOException {
return ESFileStore.getMatchingFileStore(path, fileStores);
}
/**
* Returns true if the path is writable.
* Acts just like {@link Files#isWritable(Path)}, except won't
* falsely return false for paths on SUBST'd drive letters
* See https://bugs.openjdk.java.net/browse/JDK-8034057
* Note this will set the file modification time (to its already-set value)
* to test access.
*/
@SuppressForbidden(reason = "works around https://bugs.openjdk.java.net/browse/JDK-8034057")
public static boolean isWritable(Path path) throws IOException {
boolean v = Files.isWritable(path);
if (v || Constants.WINDOWS == false) {
return v;
}
// isWritable returned false on windows, the hack begins!!!!!!
// resetting the modification time is the least destructive/simplest
// way to check for both files and directories, and fails early just
// in getting the current value if file doesn't exist, etc
try {
Files.setLastModifiedTime(path, Files.getLastModifiedTime(path));
return true;
} catch (Throwable e) {
return false;
}
}
}

4
core/src/main/java/org/elasticsearch/plugins/PluginManager.java
View File

@ -112,7 +112,7 @@ public class PluginManager {
Files.createDirectory(environment.pluginsFile());
}
if (!Files.isWritable(environment.pluginsFile())) {
if (!Environment.isWritable(environment.pluginsFile())) {
throw new IOException("plugin directory " + environment.pluginsFile() + " is read only");
}
@ -246,7 +246,7 @@ public class PluginManager {
} catch (IOException e) {
throw new IOException("Could not move [" + binFile + "] to [" + toLocation + "]", e);
}
if (Files.getFileStore(toLocation).supportsFileAttributeView(PosixFileAttributeView.class)) {
if (Environment.getFileStore(toLocation).supportsFileAttributeView(PosixFileAttributeView.class)) {
// add read and execute permissions to existing perms, so execution will work.
// read should generally be set already, but set it anyway: don't rely on umask...
final Set<PosixFilePermission> executePerms = new HashSet<>();

11
core/src/main/java/org/elasticsearch/plugins/PluginsService.java
View File

@ -200,9 +200,18 @@ public class PluginsService extends AbstractComponent {
}
public Settings updatedSettings() {
Map<String, String> foundSettings = new HashMap<>();
final Settings.Builder builder = Settings.settingsBuilder();
for (Tuple<PluginInfo, Plugin> plugin : plugins) {
builder.put(plugin.v2().additionalSettings());
Settings settings = plugin.v2().additionalSettings();
for (String setting : settings.getAsMap().keySet()) {
String oldPlugin = foundSettings.put(setting, plugin.v1().getName());
if (oldPlugin != null) {
throw new IllegalArgumentException("Cannot have additional setting [" + setting + "] " +
"in plugin [" + plugin.v1().getName() + "], already added in plugin [" + oldPlugin + "]");
}
}
builder.put(settings);
}
return builder.put(this.settings).build();
}

37
core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
View File

@ -19,44 +19,33 @@
package org.elasticsearch.repositories;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.elasticsearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.index.snapshots.IndexShardRepository;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.repositories.fs.FsRepositoryModule;
import org.elasticsearch.repositories.uri.URLRepository;
import org.elasticsearch.repositories.uri.URLRepositoryModule;
import org.elasticsearch.snapshots.RestoreService;
import org.elasticsearch.snapshots.SnapshotsService;
import org.elasticsearch.snapshots.SnapshotShardsService;
import java.util.Map;
import org.elasticsearch.snapshots.SnapshotsService;
/**
* Module responsible for registering other repositories.
* <p/>
* Repositories implemented as plugins should implement {@code onModule(RepositoriesModule module)} method, in which
* they should register repository using {@link #registerRepository(String, Class)} method.
* Sets up classes for Snapshot/Restore.
*
* Plugins can add custom repository types by calling {@link #registerRepository(String, Class, Class)}.
*/
public class RepositoriesModule extends AbstractModule {
private Map<String, Class<? extends Module>> repositoryTypes = Maps.newHashMap();
private final RepositoryTypesRegistry repositoryTypes = new RepositoryTypesRegistry();
public RepositoriesModule() {
registerRepository(FsRepository.TYPE, FsRepositoryModule.class);
registerRepository(URLRepository.TYPE, URLRepositoryModule.class);
registerRepository(FsRepository.TYPE, FsRepository.class, BlobStoreIndexShardRepository.class);
registerRepository(URLRepository.TYPE, URLRepository.class, BlobStoreIndexShardRepository.class);
}
/**
* Registers a custom repository type name against a module.
*
* @param type The type
* @param module The module
*/
public void registerRepository(String type, Class<? extends Module> module) {
repositoryTypes.put(type, module);
/** Registers a custom repository type to the given {@link Repository} and {@link IndexShardRepository}. */
public void registerRepository(String type, Class<? extends Repository> repositoryType, Class<? extends IndexShardRepository> shardRepositoryType) {
repositoryTypes.registerRepository(type, repositoryType, shardRepositoryType);
}
@Override
@ -66,6 +55,6 @@ public class RepositoriesModule extends AbstractModule {
bind(SnapshotShardsService.class).asEagerSingleton();
bind(TransportNodesSnapshotsStatus.class).asEagerSingleton();
bind(RestoreService.class).asEagerSingleton();
bind(RepositoryTypesRegistry.class).toInstance(new RepositoryTypesRegistry(ImmutableMap.copyOf(repositoryTypes)));
bind(RepositoryTypesRegistry.class).toInstance(repositoryTypes);
}
}

25
core/src/main/java/org/elasticsearch/repositories/RepositoryModule.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.repositories;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.Modules;
@ -29,12 +28,10 @@ import org.elasticsearch.common.settings.Settings;
import java.util.Arrays;
import java.util.Collections;
import static org.elasticsearch.common.Strings.toCamelCase;
/**
* This module spawns specific repository module
* Binds repository classes for the specific repository type.
*/
public class RepositoryModule extends AbstractModule implements SpawnModules {
public class RepositoryModule extends AbstractModule {
private RepositoryName repositoryName;
@ -59,28 +56,12 @@ public class RepositoryModule extends AbstractModule implements SpawnModules {
this.typesRegistry = typesRegistry;
}
/**
* Returns repository module.
* <p/>
* First repository type is looked up in typesRegistry and if it's not found there, this module tries to
* load repository by it's class name.
*
* @return repository module
*/
@Override
public Iterable<? extends Module> spawnModules() {
Class<? extends Module> repoModuleClass = typesRegistry.type(repositoryName.type());
if (repoModuleClass == null) {
throw new IllegalArgumentException("Could not find repository type [" + repositoryName.getType() + "] for repository [" + repositoryName.getName() + "]");
}
return Collections.unmodifiableList(Arrays.asList(Modules.createModule(repoModuleClass, globalSettings)));
}
/**
* {@inheritDoc}
*/
@Override
protected void configure() {
typesRegistry.bindType(binder(), repositoryName.type());
bind(RepositorySettings.class).toInstance(new RepositorySettings(globalSettings, settings));
}
}

37
core/src/main/java/org/elasticsearch/repositories/RepositoryTypesRegistry.java
View File

@ -19,31 +19,34 @@
package org.elasticsearch.repositories;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.Binder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.ExtensionPoint;
import org.elasticsearch.index.snapshots.IndexShardRepository;
/**
* Map of registered repository types and associated with these types modules
* A mapping from type name to implementations of {@link Repository} and {@link IndexShardRepository}.
*/
public class RepositoryTypesRegistry {
private final ImmutableMap<String, Class<? extends Module>> repositoryTypes;
// invariant: repositories and shardRepositories have the same keyset
private final ExtensionPoint.SelectedType<Repository> repositoryTypes =
new ExtensionPoint.SelectedType<>("repository", Repository.class);
private final ExtensionPoint.SelectedType<IndexShardRepository> shardRepositoryTypes =
new ExtensionPoint.SelectedType<>("index_repository", IndexShardRepository.class);
/**
* Creates new repository with given map of types
*
* @param repositoryTypes
*/
public RepositoryTypesRegistry(ImmutableMap<String, Class<? extends Module>> repositoryTypes) {
this.repositoryTypes = repositoryTypes;
/** Adds a new repository type to the registry, bound to the given implementation classes. */
public void registerRepository(String name, Class<? extends Repository> repositoryType, Class<? extends IndexShardRepository> shardRepositoryType) {
repositoryTypes.registerExtension(name, repositoryType);
shardRepositoryTypes.registerExtension(name, shardRepositoryType);
}
/**
* Returns repository module class for the given type
*
* @param type repository type
* @return repository module class or null if type is not found
* Looks up the given type and binds the implementation into the given binder.
* Throws an {@link IllegalArgumentException} if the given type does not exist.
*/
public Class<? extends Module> type(String type) {
return repositoryTypes.get(type);
public void bindType(Binder binder, String type) {
Settings settings = Settings.builder().put("type", type).build();
repositoryTypes.bindType(binder, settings, "type", null);
shardRepositoryTypes.bindType(binder, settings, "type", null);
}
}

46
core/src/main/java/org/elasticsearch/repositories/fs/FsRepositoryModule.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.fs;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.index.snapshots.IndexShardRepository;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
import org.elasticsearch.repositories.Repository;
/**
* File system repository module
*/
public class FsRepositoryModule extends AbstractModule {
public FsRepositoryModule() {
super();
}
/**
* {@inheritDoc}
*/
@Override
protected void configure() {
bind(Repository.class).to(FsRepository.class).asEagerSingleton();
bind(IndexShardRepository.class).to(BlobStoreIndexShardRepository.class).asEagerSingleton();
}
}

46
core/src/main/java/org/elasticsearch/repositories/uri/URLRepositoryModule.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.uri;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.index.snapshots.IndexShardRepository;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
import org.elasticsearch.repositories.Repository;
/**
* URL repository module
*/
public class URLRepositoryModule extends AbstractModule {
public URLRepositoryModule() {
super();
}
/**
* {@inheritDoc}
*/
@Override
protected void configure() {
bind(Repository.class).to(URLRepository.class).asEagerSingleton();
bind(IndexShardRepository.class).to(BlobStoreIndexShardRepository.class).asEagerSingleton();
}
}

6
core/src/test/java/org/elasticsearch/common/ParseFieldTests.java
View File

@ -18,7 +18,6 @@
*/
package org.elasticsearch.common;
import org.apache.commons.lang3.ArrayUtils;
import org.elasticsearch.test.ESTestCase;
import org.junit.Test;
@ -77,7 +76,10 @@ public class ParseFieldTests extends ESTestCase {
String[] deprecated = new String[]{"text", "same_as_text"};
String[] allValues = values;
if (withDeprecatedNames) {
allValues = ArrayUtils.addAll(values, deprecated);
String[] newArray = new String[allValues.length + deprecated.length];
System.arraycopy(allValues, 0, newArray, 0, allValues.length);
System.arraycopy(deprecated, 0, newArray, allValues.length, deprecated.length);
allValues = newArray;
}
ParseField field = new ParseField(randomFrom(values));

85
core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugins;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.store.IndexStoreModule;
import org.elasticsearch.test.ESTestCase;
public class PluginsServiceTests extends ESTestCase {
public static class AdditionalSettingsPlugin1 extends AbstractPlugin {
@Override
public String name() {
return "additional-settings1";
}
@Override
public String description() {
return "adds additional setting 'foo.bar'";
}
@Override
public Settings additionalSettings() {
return Settings.builder().put("foo.bar", "1").put(IndexStoreModule.STORE_TYPE, IndexStoreModule.Type.MMAPFS.getSettingsKey()).build();
}
}
public static class AdditionalSettingsPlugin2 extends AbstractPlugin {
@Override
public String name() {
return "additional-settings2";
}
@Override
public String description() {
return "adds additional setting 'foo.bar'";
}
@Override
public Settings additionalSettings() {
return Settings.builder().put("foo.bar", "2").build();
}
}
public void testAdditionalSettings() {
Settings settings = Settings.builder()
.put("path.home", createTempDir())
.put("my.setting", "test")
.put(IndexStoreModule.STORE_TYPE, IndexStoreModule.Type.SIMPLEFS.getSettingsKey())
.putArray("plugin.types", AdditionalSettingsPlugin1.class.getName()).build();
PluginsService service = new PluginsService(settings, new Environment(settings));
Settings newSettings = service.updatedSettings();
assertEquals("test", newSettings.get("my.setting")); // previous settings still exist
assertEquals("1", newSettings.get("foo.bar")); // added setting exists
assertEquals(IndexStoreModule.Type.SIMPLEFS.getSettingsKey(), newSettings.get(IndexStoreModule.STORE_TYPE)); // does not override pre existing settings
}
public void testAdditionalSettingsClash() {
Settings settings = Settings.builder()
.put("path.home", createTempDir())
.putArray("plugin.types", AdditionalSettingsPlugin1.class.getName(), AdditionalSettingsPlugin2.class.getName()).build();
PluginsService service = new PluginsService(settings, new Environment(settings));
try {
service.updatedSettings();
fail("Expected exception when building updated settings");
} catch (IllegalArgumentException e) {
String msg = e.getMessage();
assertTrue(msg, msg.contains("Cannot have additional setting [foo.bar]"));
assertTrue(msg, msg.contains("plugin [additional-settings1]"));
assertTrue(msg, msg.contains("plugin [additional-settings2]"));
}
}
}

9
core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
View File

@ -35,7 +35,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.snapshots.mockstore.MockRepository;
import org.elasticsearch.snapshots.mockstore.MockRepositoryPlugin;
import org.elasticsearch.test.ESIntegTestCase;
import java.io.IOException;
@ -58,10 +57,10 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return settingsBuilder().put(super.nodeSettings(nodeOrdinal))
// Rebalancing is causing some checks after restore to randomly fail
// due to https://github.com/elastic/elasticsearch/issues/9421
.put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE, EnableAllocationDecider.Rebalance.NONE)
.extendArray("plugin.types", MockRepositoryPlugin.class.getName()).build();
// Rebalancing is causing some checks after restore to randomly fail
// due to https://github.com/elastic/elasticsearch/issues/9421
.put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE, EnableAllocationDecider.Rebalance.NONE)
.extendArray("plugin.types", MockRepository.Plugin.class.getName()).build();
}
public static long getFailureCount(String repository) {

49
core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
View File

@ -21,10 +21,8 @@ package org.elasticsearch.snapshots;
import com.carrotsearch.hppc.IntHashSet;
import com.carrotsearch.hppc.IntSet;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListenableFuture;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
@ -41,8 +39,8 @@ import org.elasticsearch.cluster.AbstractDiffable;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ProcessedClusterStateUpdateTask;
import org.elasticsearch.cluster.metadata.MetaData.Custom;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.MetaData.Custom;
import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.elasticsearch.common.Nullable;
@ -64,11 +62,9 @@ import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.action.admin.cluster.repositories.get.RestGetRepositoriesAction;
import org.elasticsearch.rest.action.admin.cluster.state.RestClusterStateAction;
import org.elasticsearch.snapshots.mockstore.MockRepositoryModule;
import org.elasticsearch.snapshots.mockstore.MockRepositoryPlugin;
import org.elasticsearch.snapshots.mockstore.MockRepository;
import org.elasticsearch.test.InternalTestCluster;
import org.elasticsearch.test.rest.FakeRestRequest;
import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
@ -88,7 +84,15 @@ import static org.elasticsearch.test.ESIntegTestCase.Scope;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
import static org.hamcrest.Matchers.*;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.lessThan;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
/**
*/
@ -466,18 +470,16 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-2")
.setIndices("test-idx-all", "test-idx-none", "test-idx-some")
.setWaitForCompletion(false).setPartial(true).execute().actionGet();
awaitBusy(new Predicate<Object>() {
assertBusy(new Runnable() {
@Override
public boolean apply(Object o) {
public void run() {
SnapshotsStatusResponse snapshotsStatusResponse = client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap-2").get();
ImmutableList<SnapshotStatus> snapshotStatuses = snapshotsStatusResponse.getSnapshots();
if (snapshotStatuses.size() == 1) {
logger.trace("current snapshot status [{}]", snapshotStatuses.get(0));
return snapshotStatuses.get(0).getState().completed();
}
return false;
assertEquals(snapshotStatuses.size(), 1);
logger.trace("current snapshot status [{}]", snapshotStatuses.get(0));
assertTrue(snapshotStatuses.get(0).getState().completed());
}
});
}, 1, TimeUnit.MINUTES);
SnapshotsStatusResponse snapshotsStatusResponse = client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap-2").get();
ImmutableList<SnapshotStatus> snapshotStatuses = snapshotsStatusResponse.getSnapshots();
assertThat(snapshotStatuses.size(), equalTo(1));
@ -489,19 +491,16 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
// There is slight delay between snapshot being marked as completed in the cluster state and on the file system
// After it was marked as completed in the cluster state - we need to check if it's completed on the file system as well
awaitBusy(new Predicate<Object>() {
assertBusy(new Runnable() {
@Override
public boolean apply(Object o) {
public void run() {
GetSnapshotsResponse response = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-2").get();
assertThat(response.getSnapshots().size(), equalTo(1));
SnapshotInfo snapshotInfo = response.getSnapshots().get(0);
if (snapshotInfo.state().completed()) {
assertThat(snapshotInfo.state(), equalTo(SnapshotState.PARTIAL));
return true;
}
return false;
assertTrue(snapshotInfo.state().completed());
assertEquals(SnapshotState.PARTIAL, snapshotInfo.state());
}
});
}, 1, TimeUnit.MINUTES);
} else {
logger.info("checking snapshot completion using wait_for_completion flag");
createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-2")
@ -615,7 +614,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
@Test
public void registrationFailureTest() {
logger.info("--> start first node");
internalCluster().startNode(settingsBuilder().put("plugin.types", MockRepositoryPlugin.class.getName()));
internalCluster().startNode(settingsBuilder().put("plugin.types", MockRepository.Plugin.class.getName()));
logger.info("--> start second node");
// Make sure the first node is elected as master
internalCluster().startNode(settingsBuilder().put("node.master", false));
@ -634,7 +633,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
@Test
public void testThatSensitiveRepositorySettingsAreNotExposed() throws Exception {
Settings nodeSettings = settingsBuilder().put("plugin.types", MockRepositoryPlugin.class.getName()).build();
Settings nodeSettings = settingsBuilder().put("plugin.types", MockRepository.Plugin.class.getName()).build();
logger.info("--> start two nodes");
internalCluster().startNodesAsync(2, nodeSettings).get();
// Register mock repositories

3
core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
View File

@ -32,15 +32,12 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.repositories.RepositoryException;
import org.elasticsearch.repositories.RepositoryVerificationException;
import org.elasticsearch.snapshots.mockstore.MockRepositoryModule;
import org.elasticsearch.snapshots.mockstore.MockRepositoryPlugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Test;
import java.nio.file.Path;
import java.util.List;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
import static org.hamcrest.Matchers.containsString;

56
core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
View File

@ -19,10 +19,8 @@
package org.elasticsearch.snapshots.mockstore;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.SnapshotId;
@ -30,11 +28,17 @@ import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.snapshots.IndexShardRepository;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
import org.elasticsearch.plugins.AbstractPlugin;
import org.elasticsearch.repositories.RepositoriesModule;
import org.elasticsearch.repositories.RepositoryName;
import org.elasticsearch.repositories.RepositorySettings;
import org.elasticsearch.repositories.fs.FsRepository;
@ -46,6 +50,10 @@ import java.io.UnsupportedEncodingException;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@ -54,10 +62,48 @@ import java.util.concurrent.atomic.AtomicLong;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
/**
*/
public class MockRepository extends FsRepository {
public static class Plugin extends AbstractPlugin {
@Override
public String name() {
return "mock-repository";
}
@Override
public String description() {
return "Mock Repository";
}
public void onModule(RepositoriesModule repositoriesModule) {
repositoriesModule.registerRepository("mock", MockRepository.class, BlobStoreIndexShardRepository.class);
}
@Override
public Collection<Class<? extends Module>> modules() {
Collection<Class<? extends Module>> modules = new ArrayList<>();
modules.add(SettingsFilteringModule.class);
return modules;
}
public static class SettingsFilteringModule extends AbstractModule {
@Override
protected void configure() {
bind(SettingsFilteringService.class).asEagerSingleton();
}
}
public static class SettingsFilteringService {
@Inject
public SettingsFilteringService(SettingsFilter settingsFilter) {
settingsFilter.addFilter("secret.mock.password");
}
}
}
private final AtomicLong failureCounter = new AtomicLong();
public long getFailureCount() {

42
core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepositoryModule.java
View File

@ -1,42 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.snapshots.mockstore;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.index.snapshots.IndexShardRepository;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
import org.elasticsearch.repositories.Repository;
/**
*/
public class MockRepositoryModule extends AbstractModule {
public MockRepositoryModule() {
super();
}
@Override
protected void configure() {
bind(Repository.class).to(MockRepository.class).asEagerSingleton();
bind(IndexShardRepository.class).to(BlobStoreIndexShardRepository.class).asEagerSingleton();
}
}

68
core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepositoryPlugin.java
View File

@ -1,68 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.snapshots.mockstore;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.repositories.RepositoriesModule;
import java.util.Collection;
import java.util.Collections;
public class MockRepositoryPlugin extends Plugin {
@Override
public String name() {
return "mock-repository";
}
@Override
public String description() {
return "Mock Repository";
}
public void onModule(RepositoriesModule repositoriesModule) {
repositoriesModule.registerRepository("mock", MockRepositoryModule.class);
}
@Override
public Collection<Module> nodeModules() {
return Collections.<Module>singletonList(new SettingsFilteringModule());
}
public static class SettingsFilteringModule extends AbstractModule {
@Override
protected void configure() {
bind(SettingsFilteringService.class).asEagerSingleton();
}
}
public static class SettingsFilteringService {
@Inject
public SettingsFilteringService(SettingsFilter settingsFilter) {
settingsFilter.addFilter("secret.mock.password");
}
}
}

6
core/src/test/java/org/elasticsearch/test/ESBackcompatTestCase.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.test;
import com.carrotsearch.randomizedtesting.annotations.TestGroup;
import org.apache.commons.lang3.ArrayUtils;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
@ -32,13 +31,8 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.indices.recovery.RecoverySettings;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.test.junit.listeners.LoggingListener;
import org.elasticsearch.test.transport.AssertingLocalTransport;
import org.elasticsearch.test.transport.MockTransportService;
import org.elasticsearch.transport.Transport;
import org.elasticsearch.transport.TransportModule;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.transport.netty.NettyTransport;
import org.junit.Ignore;
import java.io.IOException;
import java.lang.annotation.ElementType;

9
core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java
View File

@ -28,12 +28,10 @@ import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.impl.client.HttpClients;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.shard.MergeSchedulerConfig;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@ -80,7 +78,6 @@ import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
@ -1920,7 +1917,11 @@ public abstract class ESIntegTestCase extends ESTestCase {
}
if (list.length != 1) {
throw new IllegalStateException("Backwards index must contain exactly one cluster\n" + StringUtils.join(list, "\n"));
StringBuilder builder = new StringBuilder("Backwards index must contain exactly one cluster\n");
for (Path line : list) {
builder.append(line.toString()).append('\n');
}
throw new IllegalStateException(builder.toString());
}
Path src = list[0];
Path dest = dataDir.resolve(internalCluster().getClusterName());

19
dev-tools/src/main/resources/ant/integration-tests.xml
View File

@ -93,21 +93,11 @@
</nested>
</run-script>
<!-- check that plugin was installed into correct place -->
<local name="longname"/>
<property name="longname" value="@{name}"/>
<local name="shortname"/>
<filter-property src="longname" dest="shortname">
<chain>
<replaceregex pattern="^elasticsearch-" replace=""/>
</chain>
</filter-property>
<fail message="did not find plugin installed as ${shortname}">
<fail message="did not find plugin installed as @{name}">
<condition>
<not>
<resourceexists>
<file file="@{home}/plugins/${shortname}"/>
<file file="@{home}/plugins/@{name}"/>
</resourceexists>
</not>
</condition>
@ -203,8 +193,8 @@
</macrodef>
<!-- Takes a plugin zip file and return the plugin name. For instance
'elasticsearch-analysis-icu-2.0.0.zip' would return
'elasticsearch-analysis-icu'. -->
'analysis-icu-2.0.0.zip' would return
'analysis-icu'. -->
<macrodef name="convert-plugin-name">
<attribute name="file"/>
<attribute name="outputproperty"/>
@ -213,7 +203,6 @@
<basename file="@{file}" property="file.base"/>
<filter-property src="file.base" dest="@{outputproperty}">
<chain>
<replaceregex pattern="^elasticsearch-" replace=""/>
<replacestring from="-${elasticsearch.version}.zip" to=""/>
</chain>
</filter-property>

3
dev-tools/src/main/resources/forbidden/all-signatures.txt
View File

@ -56,3 +56,6 @@ java.io.ObjectInputStream
java.io.ObjectInput
java.nio.file.Files#isHidden(java.nio.file.Path) @ Dependent on the operating system, use FileSystemUtils.isHidden instead
java.nio.file.Files#getFileStore(java.nio.file.Path) @ Use Environment.getFileStore() instead, impacted by JDK-8034057
java.nio.file.Files#isWritable(java.nio.file.Path) @ Use Environment.isWritable() instead, impacted by JDK-8034057

6
docs/plugins/api.asciidoc
View File

@ -15,12 +15,6 @@ The delete by query plugin adds support for deleting all of the documents
replacement for the problematic _delete-by-query_ functionality which has been
removed from Elasticsearch core.
https://github.com/elasticsearch/elasticsearch-mapper-attachments[Mapper Attachments Type plugin]::
Integrates http://lucene.apache.org/tika/[Apache Tika] to provide a new field
type `attachment` to allow indexing of documents such as PDFs and Microsoft
Word.
[float]
=== Community contributed API extension plugins

4
docs/plugins/cloud-aws.asciidoc
View File

@ -259,8 +259,8 @@ The following settings are supported:
`base_path`::
Specifies the path within bucket to repository data. Defaults to root
directory.
Specifies the path within bucket to repository data. Defaults to
value of `repositories.s3.base_path` or to root directory if not set.
`access_key`::

4
docs/plugins/cloud-azure.asciidoc
View File

@ -13,7 +13,7 @@ This plugin can be installed using the plugin manager:
[source,sh]
----------------------------------------------------------------
sudo bin/plugin install cloud-aws
sudo bin/plugin install cloud-azure
----------------------------------------------------------------
The plugin must be installed on every node in the cluster, and each node must
@ -27,7 +27,7 @@ The plugin can be removed with the following command:
[source,sh]
----------------------------------------------------------------
sudo bin/plugin remove cloud-aws
sudo bin/plugin remove cloud-azure
----------------------------------------------------------------
The node must be stopped before removing the plugin.

6
docs/plugins/mapper.asciidoc
View File

@ -8,6 +8,12 @@ Mapper plugins allow new field datatypes to be added to Elasticsearch.
The core mapper plugins are:
https://github.com/elasticsearch/elasticsearch-mapper-attachments[Mapper Attachments Type plugin]::
Integrates http://lucene.apache.org/tika/[Apache Tika] to provide a new field
type `attachment` to allow indexing of documents such as PDFs and Microsoft
Word.
<<mapper-size>>::
The mapper-size plugin provides the `_size` meta field which, when enabled,

15
docs/plugins/plugin-script.asciidoc
View File

@ -223,18 +223,3 @@ plugin.mandatory: mapper-attachments,lang-groovy
For safety reasons, a node will not start if it is missing a mandatory plugin.
[float]
=== Lucene version dependent plugins
For some plugins, such as analysis plugins, a specific major Lucene version is
required to run. In that case, the plugin provides in its
`es-plugin.properties` file the Lucene version for which the plugin was built for.
If present at startup the node will check the Lucene version before loading
the plugin. You can disable that check using
[source,yaml]
--------------------------------------------------
plugins.check_lucene: false
--------------------------------------------------

5
docs/reference/migration/migrate_2_0/removals.asciidoc
View File

@ -16,6 +16,11 @@ Facets, deprecated since 1.0, have now been removed. Instead, use the much
more powerful and flexible <<search-aggregations,aggregations>> framework.
This also means that Kibana 3 will not work with Elasticsearch 2.0.
==== MVEL has been removed
The MVEL scripting language has been removed. The default scripting language
is now Groovy.
==== Delete-by-query is now a plugin
The old delete-by-query functionality was fast but unsafe. It could lead to

4
plugins/cloud-aws/src/main/java/org/elasticsearch/plugin/cloud/aws/CloudAwsPlugin.java
View File

@ -27,9 +27,9 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.discovery.DiscoveryModule;
import org.elasticsearch.discovery.ec2.Ec2Discovery;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
import org.elasticsearch.repositories.RepositoriesModule;
import org.elasticsearch.repositories.s3.S3Repository;
import org.elasticsearch.repositories.s3.S3RepositoryModule;
import java.util.ArrayList;
import java.util.Collection;
@ -76,7 +76,7 @@ public class CloudAwsPlugin extends Plugin {
public void onModule(RepositoriesModule repositoriesModule) {
if (settings.getAsBoolean("cloud.enabled", true)) {
repositoriesModule.registerRepository(S3Repository.TYPE, S3RepositoryModule.class);
repositoriesModule.registerRepository(S3Repository.TYPE, S3Repository.class, BlobStoreIndexShardRepository.class);
}
}

45
plugins/cloud-aws/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryModule.java
View File

@ -1,45 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.s3;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.index.snapshots.IndexShardRepository;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
import org.elasticsearch.repositories.Repository;
/**
* S3 repository module
*/
public class S3RepositoryModule extends AbstractModule {
public S3RepositoryModule() {
super();
}
/**
* {@inheritDoc}
*/
@Override
protected void configure() {
bind(Repository.class).to(S3Repository.class).asEagerSingleton();
bind(IndexShardRepository.class).to(BlobStoreIndexShardRepository.class).asEagerSingleton();
}
}

0
distribution/licenses/commons-lang3-3.3.2.jar.sha1 → plugins/cloud-azure/licenses/commons-lang3-3.3.2.jar.sha1
View File

0
distribution/licenses/commons-lang3-LICENSE.txt → plugins/cloud-azure/licenses/commons-lang3-LICENSE.txt
View File

0
distribution/licenses/commons-lang3-NOTICE.txt → plugins/cloud-azure/licenses/commons-lang3-NOTICE.txt
View File

2
plugins/cloud-azure/src/main/java/org/elasticsearch/cloud/azure/AzureModule.java
View File

@ -27,6 +27,7 @@ import org.elasticsearch.cloud.azure.management.AzureComputeSettingsFilter;
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
import org.elasticsearch.cloud.azure.storage.AzureStorageSettingsFilter;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Inject;
@ -73,6 +74,7 @@ public class AzureModule extends AbstractModule {
@Override
protected void configure() {
logger.debug("starting azure services");
bind(AzureStorageSettingsFilter.class).asEagerSingleton();
bind(AzureComputeSettingsFilter.class).asEagerSingleton();
// If we have set discovery to azure, let's start the azure compute service

8
plugins/cloud-azure/src/main/java/org/elasticsearch/plugin/cloud/azure/CloudAzurePlugin.java
View File

@ -26,18 +26,20 @@ import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.discovery.DiscoveryModule;
import org.elasticsearch.discovery.azure.AzureDiscovery;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
import org.elasticsearch.index.store.IndexStoreModule;
import org.elasticsearch.index.store.smbmmapfs.SmbMmapFsIndexStore;
import org.elasticsearch.index.store.smbsimplefs.SmbSimpleFsIndexStore;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.repositories.RepositoriesModule;
import org.elasticsearch.repositories.azure.AzureRepository;
import org.elasticsearch.repositories.azure.AzureRepositoryModule;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import static org.elasticsearch.cloud.azure.AzureModule.isSnapshotReady;
/**
*
*/
@ -71,7 +73,9 @@ public class CloudAzurePlugin extends Plugin {
}
public void onModule(RepositoriesModule module) {
module.registerRepository(AzureRepository.TYPE, AzureRepositoryModule.class);
if (isSnapshotReady(settings, logger)) {
module.registerRepository(AzureRepository.TYPE, AzureRepository.class, BlobStoreIndexShardRepository.class);
}
}
public void onModule(DiscoveryModule discoveryModule) {

61
plugins/cloud-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryModule.java
View File

@ -1,61 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.azure;
import org.elasticsearch.cloud.azure.AzureModule;
import org.elasticsearch.cloud.azure.storage.AzureStorageSettingsFilter;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.snapshots.IndexShardRepository;
import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
import org.elasticsearch.repositories.Repository;
/**
* Azure repository module
*/
public class AzureRepositoryModule extends AbstractModule {
protected final ESLogger logger;
private Settings settings;
public AzureRepositoryModule(Settings settings) {
super();
this.logger = Loggers.getLogger(getClass(), settings);
this.settings = settings;
}
/**
* {@inheritDoc}
*/
@Override
protected void configure() {
bind(AzureStorageSettingsFilter.class).asEagerSingleton();
if (AzureModule.isSnapshotReady(settings, logger)) {
bind(Repository.class).to(AzureRepository.class).asEagerSingleton();
bind(IndexShardRepository.class).to(BlobStoreIndexShardRepository.class).asEagerSingleton();
} else {
logger.debug("disabling azure snapshot and restore features");
}
}
}

5
plugins/pom.xml
View File

@ -177,11 +177,6 @@
<artifactId>t-digest</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>

6
pom.xml
View File

@ -409,12 +409,6 @@
<version>2.1.6</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.3.2</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>

5
qa/smoke-test-multinode/pom.xml
View File

@ -165,11 +165,6 @@
<artifactId>t-digest</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>

5
qa/smoke-test-plugins/pom.xml
View File

@ -170,11 +170,6 @@
<artifactId>t-digest</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>

2
qa/vagrant/pom.xml
View File

@ -94,7 +94,7 @@
</artifactItem>
<artifactItem>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>elasticsearch-jvm-example</artifactId>
<artifactId>jvm-example</artifactId>
<version>${elasticsearch.version}</version>
<type>zip</type>
</artifactItem>

12
qa/vagrant/src/test/resources/packaging/scripts/25_tar_plugins.bats
View File

@ -61,7 +61,7 @@ setup() {
assert_file_exist "/tmp/elasticsearch/config/jvm-example/example.yaml"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/plugin-descriptor.properties"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/elasticsearch-jvm-example-"*".jar"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/jvm-example-"*".jar"
echo "Running jvm-example's bin script...."
/tmp/elasticsearch/bin/jvm-example/test | grep test
@ -106,7 +106,7 @@ setup() {
assert_file_exist "/tmp/elasticsearch/config/jvm-example/example.yaml"
assert_file_exist "$TEMP_PLUGINS_DIR/jvm-example"
assert_file_exist "$TEMP_PLUGINS_DIR/jvm-example/plugin-descriptor.properties"
assert_file_exist "$TEMP_PLUGINS_DIR/jvm-example/elasticsearch-jvm-example-"*".jar"
assert_file_exist "$TEMP_PLUGINS_DIR/jvm-example/jvm-example-"*".jar"
# Remove the plugin
run /tmp/elasticsearch/bin/plugin remove jvm-example
@ -156,7 +156,7 @@ setup() {
assert_file_exist "$TEMP_CONFIG_DIR/jvm-example/example.yaml"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/plugin-descriptor.properties"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/elasticsearch-jvm-example-"*".jar"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/jvm-example-"*".jar"
# Remove the plugin
run /tmp/elasticsearch/bin/plugin remove jvm-example
@ -210,7 +210,7 @@ setup() {
assert_file_exist "$TEMP_CONFIG_DIR/jvm-example/example.yaml"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/plugin-descriptor.properties"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/elasticsearch-jvm-example-"*".jar"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/jvm-example-"*".jar"
# Remove the plugin
run /tmp/elasticsearch/bin/plugin remove jvm-example
@ -254,7 +254,7 @@ setup() {
assert_file_exist "$ES_DIR/config/jvm-example/example.yaml"
assert_file_exist "$ES_DIR/plugins/jvm-example"
assert_file_exist "$ES_DIR/plugins/jvm-example/plugin-descriptor.properties"
assert_file_exist "$ES_DIR/plugins/jvm-example/elasticsearch-jvm-example-"*".jar"
assert_file_exist "$ES_DIR/plugins/jvm-example/jvm-example-"*".jar"
# Remove the plugin
run "$ES_DIR/bin/plugin" remove jvm-example
@ -298,7 +298,7 @@ setup() {
assert_file_exist "/tmp/elasticsearch/config/jvm-example/example.yaml"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/plugin-descriptor.properties"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/elasticsearch-jvm-example-"*".jar"
assert_file_exist "/tmp/elasticsearch/plugins/jvm-example/jvm-example-"*".jar"
# Remove the plugin
run /tmp/elasticsearch/bin/plugin remove jvm-example

8
qa/vagrant/src/test/resources/packaging/scripts/50_plugins.bats
View File

@ -74,7 +74,7 @@ install_package() {
assert_file_exist "/etc/elasticsearch/jvm-example/example.yaml"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example/plugin-descriptor.properties"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example/elasticsearch-jvm-example-"*".jar"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example/jvm-example-"*".jar"
# Remove the plugin
run /usr/share/elasticsearch/bin/plugin remove jvm-example
@ -121,7 +121,7 @@ install_package() {
assert_file_exist "/etc/elasticsearch/jvm-example/example.yaml"
assert_file_exist "$TEMP_PLUGINS_DIR/jvm-example"
assert_file_exist "$TEMP_PLUGINS_DIR/jvm-example/plugin-descriptor.properties"
assert_file_exist "$TEMP_PLUGINS_DIR/jvm-example/elasticsearch-jvm-example-"*".jar"
assert_file_exist "$TEMP_PLUGINS_DIR/jvm-example/jvm-example-"*".jar"
# Remove the plugin
@ -183,7 +183,7 @@ install_package() {
assert_file_exist "$TEMP_CONFIG_DIR/jvm-example/example.yaml"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example/plugin-descriptor.properties"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example/elasticsearch-jvm-example-"*".jar"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example/jvm-example-"*".jar"
# Remove the plugin
run /usr/share/elasticsearch/bin/plugin remove jvm-example
@ -241,7 +241,7 @@ install_package() {
assert_file_exist "$TEMP_CONFIG_DIR/jvm-example/example.yaml"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example/plugin-descriptor.properties"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example/elasticsearch-jvm-example-"*".jar"
assert_file_exist "/usr/share/elasticsearch/plugins/jvm-example/jvm-example-"*".jar"
# Remove the plugin
run /usr/share/elasticsearch/bin/plugin remove jvm-example

2
qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
View File

@ -27,7 +27,7 @@
# Variables used by tests
EXAMPLE_PLUGIN_ZIP=$(readlink -m elasticsearch-jvm-example-*.zip)
EXAMPLE_PLUGIN_ZIP=$(readlink -m jvm-example-*.zip)
# Checks if necessary commands are available to run the tests