honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into feature/client_aggs_parsing

This commit is contained in:
Tanguy Leroux 2017-04-18 14:55:58 +02:00
parent c0036d8516 f217eb8ad8
commit 67a9696e55
145 changed files with 2561 additions and 1794 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.dir-locals.el
View File

@ -83,6 +83,6 @@
))
(c-basic-offset . 4)
(c-comment-only-line-offset . (0 . 0))
(fill-column . 100)
(fci-rule-column . 100)
(fill-column . 140)
(fci-rule-column . 140)
(compile-command . "gradle compileTestJava"))))

7
TESTING.asciidoc
View File

@ -362,7 +362,7 @@ quality boxes available in vagrant atlas:
* sles-11
We're missing the follow because our tests are very linux/bash centric:
We're missing the following because our tests are very linux/bash centric:
* Windows Server 2012
@ -418,8 +418,8 @@ sudo -E bats $BATS_TESTS/*rpm*.bats
If you wanted to retest all the release artifacts on a single VM you could:
-------------------------------------------------
gradle vagrantSetUp
vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404
gradle setupBats
cd qa/vagrant; vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404
cd $BATS_ARCHIVES
sudo -E bats $BATS_TESTS/*.bats
-------------------------------------------------
@ -505,4 +505,3 @@ included as part of the build by checking the projects of the build.
---------------------------------------------------------------------------
gradle projects
---------------------------------------------------------------------------

10
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
View File

@ -88,7 +88,7 @@ class ClusterConfiguration {
if (seedNode == node) {
return null
}
ant.waitfor(maxwait: '20', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond') {
ant.waitfor(maxwait: '40', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond') {
resourceexists {
file(file: seedNode.transportPortsFile.toString())
}
@ -136,6 +136,8 @@ class ClusterConfiguration {
LinkedHashMap<String, Object[]> setupCommands = new LinkedHashMap<>()
List<Object> dependencies = new ArrayList<>()
@Input
void systemProperty(String property, String value) {
systemProperties.put(property, value)
@ -179,4 +181,10 @@ class ClusterConfiguration {
}
extraConfigFiles.put(path, sourceFile)
}
/** Add dependencies that must be run before the first task setting up the cluster. */
@Input
void dependsOn(Object... deps) {
dependencies.addAll(deps)
}
}

18
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View File

@ -57,7 +57,7 @@ class ClusterFormationTasks {
// first we remove everything in the shared cluster directory to ensure there are no leftovers in repos or anything
// in theory this should not be necessary but repositories are only deleted in the cluster-state and not on-disk
// such that snapshots survive failures / test runs and there is no simple way today to fix that.
Task cleanup = project.tasks.create(name: "${prefix}#prepareCluster.cleanShared", type: Delete, dependsOn: runner.dependsOn.collect()) {
Task cleanup = project.tasks.create(name: "${prefix}#prepareCluster.cleanShared", type: Delete, dependsOn: config.dependencies) {
delete sharedDir
doLast {
sharedDir.mkdirs()
@ -104,7 +104,7 @@ class ClusterFormationTasks {
NodeInfo node = new NodeInfo(config, i, project, prefix, elasticsearchVersion, sharedDir)
nodes.add(node)
Task dependsOn = startTasks.empty ? cleanup : startTasks.get(0)
startTasks.add(configureNode(project, prefix, runner, dependsOn, node, distro, nodes.get(0)))
startTasks.add(configureNode(project, prefix, runner, dependsOn, node, config, distro, nodes.get(0)))
}
Task wait = configureWaitTask("${prefix}#wait", project, nodes, startTasks)
@ -148,7 +148,8 @@ class ClusterFormationTasks {
*
* @return a task which starts the node.
*/
static Task configureNode(Project project, String prefix, Task runner, Object dependsOn, NodeInfo node, Configuration configuration, NodeInfo seedNode) {
static Task configureNode(Project project, String prefix, Task runner, Object dependsOn, NodeInfo node, ClusterConfiguration config,
Configuration distribution, NodeInfo seedNode) {
// tasks are chained so their execution order is maintained
Task setup = project.tasks.create(name: taskName(prefix, node, 'clean'), type: Delete, dependsOn: dependsOn) {
@ -161,7 +162,7 @@ class ClusterFormationTasks {
setup = configureCheckPreviousTask(taskName(prefix, node, 'checkPrevious'), project, setup, node)
setup = configureStopTask(taskName(prefix, node, 'stopPrevious'), project, setup, node)
setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, configuration)
setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distribution)
setup = configureWriteConfigTask(taskName(prefix, node, 'configure'), project, setup, node, seedNode)
setup = configureCreateKeystoreTask(taskName(prefix, node, 'createKeystore'), project, setup, node)
setup = configureAddKeystoreSettingTasks(prefix, project, setup, node)
@ -205,6 +206,13 @@ class ClusterFormationTasks {
// if we are running in the background, make sure to stop the server when the task completes
runner.finalizedBy(stop)
start.finalizedBy(stop)
for (Object dependency : config.dependencies) {
if (dependency instanceof Fixture) {
Task depStop = ((Fixture)dependency).stopTask
runner.finalizedBy(depStop)
start.finalizedBy(depStop)
}
}
}
return start
}
@ -541,7 +549,7 @@ class ClusterFormationTasks {
static Task configureWaitTask(String name, Project project, List<NodeInfo> nodes, List<Task> startTasks) {
Task wait = project.tasks.create(name: name, dependsOn: startTasks)
wait.doLast {
ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${name}") {
ant.waitfor(maxwait: '60', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${name}") {
or {
for (NodeInfo node : nodes) {
resourceexists {

4
buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
View File

@ -477,10 +477,10 @@ class VagrantTestPlugin implements Plugin<Project> {
}
}
}
packaging.doFirst {
platform.doFirst {
project.gradle.addListener(platformReproListener)
}
packaging.doLast {
platform.doLast {
project.gradle.removeListener(platformReproListener)
}
if (project.extensions.esvagrant.boxes.contains(box)) {

2
client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java
View File

@ -308,7 +308,7 @@ public class CrudIT extends ESRestHighLevelClientTestCase {
assertEquals(RestStatus.BAD_REQUEST, exception.status());
assertEquals("Elasticsearch exception [type=illegal_argument_exception, " +
"reason=Can't specify parent if no parent field has been configured]", exception.getMessage());
"reason=can't specify parent if no parent field has been configured]", exception.getMessage());
}
{
ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> {

7
core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
View File

@ -184,12 +184,7 @@ public class BulkProcessor implements Closeable {
this.bulkActions = bulkActions;
this.bulkSize = bulkSize.getBytes();
this.bulkRequest = new BulkRequest();
if (concurrentRequests == 0) {
this.bulkRequestHandler = BulkRequestHandler.syncHandler(consumer, backoffPolicy, listener, threadPool);
} else {
this.bulkRequestHandler = BulkRequestHandler.asyncHandler(consumer, backoffPolicy, listener, threadPool, concurrentRequests);
}
this.bulkRequestHandler = new BulkRequestHandler(consumer, backoffPolicy, listener, threadPool, concurrentRequests);
// Start period flushing task after everything is setup
this.cancellableFlushTask = startFlushTask(flushInterval, threadPool);

195
core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java
View File

@ -27,155 +27,86 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.elasticsearch.threadpool.ThreadPool;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
/**
* Abstracts the low-level details of bulk request handling
* Implements the low-level details of bulk request handling
*/
abstract class BulkRequestHandler {
protected final Logger logger;
protected final BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer;
protected final ThreadPool threadPool;
public final class BulkRequestHandler {
private final Logger logger;
private final BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer;
private final BulkProcessor.Listener listener;
private final Semaphore semaphore;
private final Retry retry;
private final int concurrentRequests;
protected BulkRequestHandler(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, ThreadPool threadPool) {
BulkRequestHandler(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BackoffPolicy backoffPolicy,
BulkProcessor.Listener listener, ThreadPool threadPool,
int concurrentRequests) {
assert concurrentRequests >= 0;
this.logger = Loggers.getLogger(getClass());
this.consumer = consumer;
this.threadPool = threadPool;
this.listener = listener;
this.concurrentRequests = concurrentRequests;
this.retry = new Retry(EsRejectedExecutionException.class, backoffPolicy, threadPool);
this.semaphore = new Semaphore(concurrentRequests > 0 ? concurrentRequests : 1);
}
public abstract void execute(BulkRequest bulkRequest, long executionId);
public abstract boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException;
public static BulkRequestHandler syncHandler(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer,
BackoffPolicy backoffPolicy, BulkProcessor.Listener listener,
ThreadPool threadPool) {
return new SyncBulkRequestHandler(consumer, backoffPolicy, listener, threadPool);
}
public static BulkRequestHandler asyncHandler(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer,
BackoffPolicy backoffPolicy, BulkProcessor.Listener listener,
ThreadPool threadPool, int concurrentRequests) {
return new AsyncBulkRequestHandler(consumer, backoffPolicy, listener, threadPool, concurrentRequests);
}
private static class SyncBulkRequestHandler extends BulkRequestHandler {
private final BulkProcessor.Listener listener;
private final BackoffPolicy backoffPolicy;
SyncBulkRequestHandler(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BackoffPolicy backoffPolicy,
BulkProcessor.Listener listener, ThreadPool threadPool) {
super(consumer, threadPool);
this.backoffPolicy = backoffPolicy;
this.listener = listener;
}
@Override
public void execute(BulkRequest bulkRequest, long executionId) {
boolean afterCalled = false;
try {
listener.beforeBulk(executionId, bulkRequest);
BulkResponse bulkResponse = Retry
.on(EsRejectedExecutionException.class)
.policy(backoffPolicy)
.using(threadPool)
.withSyncBackoff(consumer, bulkRequest, Settings.EMPTY);
afterCalled = true;
listener.afterBulk(executionId, bulkRequest, bulkResponse);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.info((Supplier<?>) () -> new ParameterizedMessage("Bulk request {} has been cancelled.", executionId), e);
if (!afterCalled) {
listener.afterBulk(executionId, bulkRequest, e);
public void execute(BulkRequest bulkRequest, long executionId) {
Runnable toRelease = () -> {};
boolean bulkRequestSetupSuccessful = false;
try {
listener.beforeBulk(executionId, bulkRequest);
semaphore.acquire();
toRelease = semaphore::release;
CountDownLatch latch = new CountDownLatch(1);
retry.withBackoff(consumer, bulkRequest, new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse response) {
try {
listener.afterBulk(executionId, bulkRequest, response);
} finally {
semaphore.release();
latch.countDown();
}
}
} catch (Exception e) {
logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to execute bulk request {}.", executionId), e);
if (!afterCalled) {
listener.afterBulk(executionId, bulkRequest, e);
@Override
public void onFailure(Exception e) {
try {
listener.afterBulk(executionId, bulkRequest, e);
} finally {
semaphore.release();
latch.countDown();
}
}
}, Settings.EMPTY);
bulkRequestSetupSuccessful = true;
if (concurrentRequests == 0) {
latch.await();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.info((Supplier<?>) () -> new ParameterizedMessage("Bulk request {} has been cancelled.", executionId), e);
listener.afterBulk(executionId, bulkRequest, e);
} catch (Exception e) {
logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to execute bulk request {}.", executionId), e);
listener.afterBulk(executionId, bulkRequest, e);
} finally {
if (bulkRequestSetupSuccessful == false) { // if we fail on client.bulk() release the semaphore
toRelease.run();
}
}
}
@Override
public boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException {
// we are "closed" immediately as there is no request in flight
boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException {
if (semaphore.tryAcquire(this.concurrentRequests, timeout, unit)) {
semaphore.release(this.concurrentRequests);
return true;
}
}
private static class AsyncBulkRequestHandler extends BulkRequestHandler {
private final BackoffPolicy backoffPolicy;
private final BulkProcessor.Listener listener;
private final Semaphore semaphore;
private final int concurrentRequests;
private AsyncBulkRequestHandler(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BackoffPolicy backoffPolicy,
BulkProcessor.Listener listener, ThreadPool threadPool,
int concurrentRequests) {
super(consumer, threadPool);
this.backoffPolicy = backoffPolicy;
assert concurrentRequests > 0;
this.listener = listener;
this.concurrentRequests = concurrentRequests;
this.semaphore = new Semaphore(concurrentRequests);
}
@Override
public void execute(BulkRequest bulkRequest, long executionId) {
boolean bulkRequestSetupSuccessful = false;
boolean acquired = false;
try {
listener.beforeBulk(executionId, bulkRequest);
semaphore.acquire();
acquired = true;
Retry.on(EsRejectedExecutionException.class)
.policy(backoffPolicy)
.using(threadPool)
.withAsyncBackoff(consumer, bulkRequest, new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse response) {
try {
listener.afterBulk(executionId, bulkRequest, response);
} finally {
semaphore.release();
}
}
@Override
public void onFailure(Exception e) {
try {
listener.afterBulk(executionId, bulkRequest, e);
} finally {
semaphore.release();
}
}
}, Settings.EMPTY);
bulkRequestSetupSuccessful = true;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.info((Supplier<?>) () -> new ParameterizedMessage("Bulk request {} has been cancelled.", executionId), e);
listener.afterBulk(executionId, bulkRequest, e);
} catch (Exception e) {
logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to execute bulk request {}.", executionId), e);
listener.afterBulk(executionId, bulkRequest, e);
} finally {
if (!bulkRequestSetupSuccessful && acquired) { // if we fail on client.bulk() release the semaphore
semaphore.release();
}
}
}
@Override
public boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException {
if (semaphore.tryAcquire(this.concurrentRequests, timeout, unit)) {
semaphore.release(this.concurrentRequests);
return true;
}
return false;
}
return false;
}
}

52
core/src/main/java/org/elasticsearch/action/bulk/Retry.java
View File

@ -25,20 +25,14 @@ import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.elasticsearch.threadpool.ThreadPool;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Predicate;
/**
@ -46,62 +40,42 @@ import java.util.function.Predicate;
*/
public class Retry {
private final Class<? extends Throwable> retryOnThrowable;
private final BackoffPolicy backoffPolicy;
private final ThreadPool threadPool;
private BackoffPolicy backoffPolicy;
private ThreadPool threadPool;
public static Retry on(Class<? extends Throwable> retryOnThrowable) {
return new Retry(retryOnThrowable);
}
Retry(Class<? extends Throwable> retryOnThrowable) {
public Retry(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, ThreadPool threadPool) {
this.retryOnThrowable = retryOnThrowable;
}
/**
* @param backoffPolicy The backoff policy that defines how long and how often to wait for retries.
*/
public Retry policy(BackoffPolicy backoffPolicy) {
this.backoffPolicy = backoffPolicy;
return this;
}
/**
* @param threadPool The threadPool that will be used to schedule retries.
*/
public Retry using(ThreadPool threadPool) {
this.threadPool = threadPool;
return this;
}
/**
* Invokes #apply(BulkRequest, ActionListener). Backs off on the provided exception and delegates results to the
* provided listener. Retries will be attempted using the provided schedule function
* Invokes #accept(BulkRequest, ActionListener). Backs off on the provided exception and delegates results to the
* provided listener. Retries will be scheduled using the class's thread pool.
* @param consumer The consumer to which apply the request and listener
* @param bulkRequest The bulk request that should be executed.
* @param listener A listener that is invoked when the bulk request finishes or completes with an exception. The listener is not
* @param settings settings
*/
public void withAsyncBackoff(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BulkRequest bulkRequest, ActionListener<BulkResponse> listener, Settings settings) {
public void withBackoff(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BulkRequest bulkRequest, ActionListener<BulkResponse> listener, Settings settings) {
RetryHandler r = new RetryHandler(retryOnThrowable, backoffPolicy, consumer, listener, settings, threadPool);
r.execute(bulkRequest);
}
/**
* Invokes #apply(BulkRequest, ActionListener). Backs off on the provided exception. Retries will be attempted using
* the provided schedule function.
* Invokes #accept(BulkRequest, ActionListener). Backs off on the provided exception. Retries will be scheduled using
* the class's thread pool.
*
* @param consumer The consumer to which apply the request and listener
* @param bulkRequest The bulk request that should be executed.
* @param settings settings
* @return the bulk response as returned by the client.
* @throws Exception Any exception thrown by the callable.
* @return a future representing the bulk response returned by the client.
*/
public BulkResponse withSyncBackoff(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BulkRequest bulkRequest, Settings settings) throws Exception {
PlainActionFuture<BulkResponse> actionFuture = PlainActionFuture.newFuture();
RetryHandler r = new RetryHandler(retryOnThrowable, backoffPolicy, consumer, actionFuture, settings, threadPool);
r.execute(bulkRequest);
return actionFuture.actionGet();
public PlainActionFuture<BulkResponse> withBackoff(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, BulkRequest bulkRequest, Settings settings) {
PlainActionFuture<BulkResponse> future = PlainActionFuture.newFuture();
withBackoff(consumer, bulkRequest, future, settings);
return future;
}
static class RetryHandler implements ActionListener<BulkResponse> {

2
core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
View File

@ -279,7 +279,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
break;
default: throw new AssertionError("request type not supported: [" + docWriteRequest.opType() + "]");
}
} catch (ElasticsearchParseException | RoutingMissingException e) {
} catch (ElasticsearchParseException | IllegalArgumentException | RoutingMissingException e) {
BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), docWriteRequest.type(), docWriteRequest.id(), e);
BulkItemResponse bulkItemResponse = new BulkItemResponse(i, docWriteRequest.opType(), failure);
responses.set(i, bulkItemResponse);

4
core/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractAsyncBulkByScrollAction.java
View File

@ -134,7 +134,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
this.mainRequest = mainRequest;
this.listener = listener;
BackoffPolicy backoffPolicy = buildBackoffPolicy();
bulkRetry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.wrap(backoffPolicy, task::countBulkRetry)).using(threadPool);
bulkRetry = new Retry(EsRejectedExecutionException.class, BackoffPolicy.wrap(backoffPolicy, task::countBulkRetry), threadPool);
scrollSource = buildScrollableResultSource(backoffPolicy);
scriptApplier = Objects.requireNonNull(buildScriptApplier(), "script applier must not be null");
/*
@ -337,7 +337,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
finishHim(null);
return;
}
bulkRetry.withAsyncBackoff(client::bulk, request, new ActionListener<BulkResponse>() {
bulkRetry.withBackoff(client::bulk, request, new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse response) {
onBulkResponse(thisBatchStartTime, response);

8
core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
View File

@ -491,14 +491,18 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
}
if (parent != null && !mappingMd.hasParentField()) {
throw new IllegalArgumentException("Can't specify parent if no parent field has been configured");
throw new IllegalArgumentException("can't specify parent if no parent field has been configured");
}
} else {
if (parent != null) {
throw new IllegalArgumentException("Can't specify parent if no parent field has been configured");
throw new IllegalArgumentException("can't specify parent if no parent field has been configured");
}
}
if ("".equals(id)) {
throw new IllegalArgumentException("if _id is specified it must not be empty");
}
// generate id if not already provided
if (id == null) {
assert autoGeneratedTimestamp == -1 : "timestamp has already been generated!";

28
core/src/main/java/org/elasticsearch/bootstrap/Security.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.bootstrap;
import org.elasticsearch.SecureSM;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.network.NetworkModule;
@ -45,11 +44,9 @@ import java.security.NoSuchAlgorithmException;
import java.security.Permissions;
import java.security.Policy;
import java.security.URIParameter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -269,6 +266,26 @@ final class Security {
for (Path path : environment.dataFiles()) {
addPath(policy, Environment.PATH_DATA_SETTING.getKey(), path, "read,readlink,write,delete");
}
/*
* If path.data and default.path.data are set, we need read access to the paths in default.path.data to check for the existence of
* index directories there that could have arisen from a bug in the handling of simultaneous configuration of path.data and
* default.path.data that was introduced in Elasticsearch 5.3.0.
*
* If path.data is not set then default.path.data would take precedence in setting the data paths for the environment and
* permissions would have been granted above.
*
* If path.data is not set and default.path.data is not set, then we would fallback to the default data directory under
* Elasticsearch home and again permissions would have been granted above.
*
* If path.data is set and default.path.data is not set, there is nothing to do here.
*/
if (Environment.PATH_DATA_SETTING.exists(environment.settings())
&& Environment.DEFAULT_PATH_DATA_SETTING.exists(environment.settings())) {
for (final String path : Environment.DEFAULT_PATH_DATA_SETTING.get(environment.settings())) {
// write permissions are not needed here, we are not going to be writing to any paths here
addPath(policy, Environment.DEFAULT_PATH_DATA_SETTING.getKey(), getPath(path), "read,readlink");
}
}
for (Path path : environment.repoFiles()) {
addPath(policy, Environment.PATH_REPO_SETTING.getKey(), path, "read,readlink,write,delete");
}
@ -278,6 +295,11 @@ final class Security {
}
}
@SuppressForbidden(reason = "read path that is not configured in environment")
private static Path getPath(final String path) {
return PathUtils.get(path);
}
/**
* Add dynamic {@link SocketPermission}s based on HTTP and transport settings.
*

8
core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java
View File

@ -30,13 +30,17 @@ import org.elasticsearch.common.util.ByteArray;
*/
public final class ReleasablePagedBytesReference extends PagedBytesReference implements Releasable {
public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int length) {
private final Releasable releasable;
public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray byteArray, int length,
Releasable releasable) {
super(bigarrays, byteArray, length);
this.releasable = releasable;
}
@Override
public void close() {
Releasables.close(byteArray);
Releasables.close(releasable);
}
}

5
core/src/main/java/org/elasticsearch/common/compress/Compressor.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.common.compress;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -31,5 +32,9 @@ public interface Compressor {
StreamInput streamInput(StreamInput in) throws IOException;
/**
* Creates a new stream output that compresses the contents and writes to the provided stream
* output. Closing the returned {@link StreamOutput} will close the provided stream output.
*/
StreamOutput streamOutput(StreamOutput out) throws IOException;
}

9
core/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.common.compress;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.Compressor;
import org.elasticsearch.common.io.stream.InputStreamStreamInput;
import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
@ -47,7 +46,7 @@ public class DeflateCompressor implements Compressor {
// It needs to be different from other compressors and to not be specific
// enough so that no stream starting with these bytes could be detected as
// a XContent
private static final byte[] HEADER = new byte[] { 'D', 'F', 'L', '\0' };
private static final byte[] HEADER = new byte[]{'D', 'F', 'L', '\0'};
// 3 is a good trade-off between speed and compression ratio
private static final int LEVEL = 3;
// We use buffering on the input and output of in/def-laters in order to
@ -88,6 +87,7 @@ public class DeflateCompressor implements Compressor {
decompressedIn = new BufferedInputStream(decompressedIn, BUFFER_SIZE);
return new InputStreamStreamInput(decompressedIn) {
final AtomicBoolean closed = new AtomicBoolean(false);
public void close() throws IOException {
try {
super.close();
@ -107,10 +107,11 @@ public class DeflateCompressor implements Compressor {
final boolean nowrap = true;
final Deflater deflater = new Deflater(LEVEL, nowrap);
final boolean syncFlush = true;
OutputStream compressedOut = new DeflaterOutputStream(out, deflater, BUFFER_SIZE, syncFlush);
compressedOut = new BufferedOutputStream(compressedOut, BUFFER_SIZE);
DeflaterOutputStream deflaterOutputStream = new DeflaterOutputStream(out, deflater, BUFFER_SIZE, syncFlush);
OutputStream compressedOut = new BufferedOutputStream(deflaterOutputStream, BUFFER_SIZE);
return new OutputStreamStreamOutput(compressedOut) {
final AtomicBoolean closed = new AtomicBoolean(false);
public void close() throws IOException {
try {
super.close();

32
core/src/main/java/org/elasticsearch/common/io/ReleasableBytesStream.java
View File

@ -1,32 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.io;
import org.elasticsearch.common.bytes.ReleasablePagedBytesReference;
/**
* A bytes stream that requires its bytes to be released once no longer used.
*/
public interface ReleasableBytesStream extends BytesStream {
@Override
ReleasablePagedBytesReference bytes();
}

55
core/src/main/java/org/elasticsearch/common/io/Streams.java
View File

@ -20,6 +20,9 @@
package org.elasticsearch.common.io;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStream;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.util.Callback;
import java.io.BufferedReader;
@ -236,4 +239,56 @@ public abstract class Streams {
}
}
}
/**
* Wraps the given {@link BytesStream} in a {@link StreamOutput} that simply flushes when
* close is called.
*/
public static BytesStream flushOnCloseStream(BytesStream os) {
return new FlushOnCloseOutputStream(os);
}
/**
* A wrapper around a {@link BytesStream} that makes the close operation a flush. This is
* needed as sometimes a stream will be closed but the bytes that the stream holds still need
* to be used and the stream cannot be closed until the bytes have been consumed.
*/
private static class FlushOnCloseOutputStream extends BytesStream {
private final BytesStream delegate;
private FlushOnCloseOutputStream(BytesStream bytesStreamOutput) {
this.delegate = bytesStreamOutput;
}
@Override
public void writeByte(byte b) throws IOException {
delegate.writeByte(b);
}
@Override
public void writeBytes(byte[] b, int offset, int length) throws IOException {
delegate.writeBytes(b, offset, length);
}
@Override
public void flush() throws IOException {
delegate.flush();
}
@Override
public void close() throws IOException {
flush();
}
@Override
public void reset() throws IOException {
delegate.reset();
}
@Override
public BytesReference bytes() {
return delegate.bytes();
}
}
}

8
core/src/main/java/org/elasticsearch/common/io/BytesStream.java → core/src/main/java/org/elasticsearch/common/io/stream/BytesStream.java
View File

@ -17,11 +17,11 @@
* under the License.
*/
package org.elasticsearch.common.io;
package org.elasticsearch.common.io.stream;
import org.elasticsearch.common.bytes.BytesReference;
public interface BytesStream {
public abstract class BytesStream extends StreamOutput {
BytesReference bytes();
}
public abstract BytesReference bytes();
}

9
core/src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.common.io.stream;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.PagedBytesReference;
import org.elasticsearch.common.io.BytesStream;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.ByteArray;
@ -31,7 +30,7 @@ import java.io.IOException;
* A @link {@link StreamOutput} that uses {@link BigArrays} to acquire pages of
* bytes, which avoids frequent reallocation &amp; copying of the internal data.
*/
public class BytesStreamOutput extends StreamOutput implements BytesStream {
public class BytesStreamOutput extends BytesStream {
protected final BigArrays bigArrays;
@ -50,7 +49,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream {
/**
* Create a non recycling {@link BytesStreamOutput} with enough initial pages acquired
* to satisfy the capacity given by expected size.
*
*
* @param expectedSize the expected maximum size of the stream in bytes.
*/
public BytesStreamOutput(int expectedSize) {
@ -129,7 +128,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream {
/**
* Returns the current size of the buffer.
*
*
* @return the value of the <code>count</code> field, which is the number of valid
* bytes in this output stream.
* @see java.io.ByteArrayOutputStream#count
@ -151,7 +150,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream {
return bytes.ramBytesUsed();
}
private void ensureCapacity(long offset) {
void ensureCapacity(long offset) {
if (offset > Integer.MAX_VALUE) {
throw new IllegalArgumentException(getClass().getSimpleName() + " cannot hold more than 2GB of data");
}

51
core/src/main/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java
View File

@ -20,29 +20,66 @@
package org.elasticsearch.common.io.stream;
import org.elasticsearch.common.bytes.ReleasablePagedBytesReference;
import org.elasticsearch.common.io.ReleasableBytesStream;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.ByteArray;
/**
* An bytes stream output that allows providing a {@link BigArrays} instance
* expecting it to require releasing its content ({@link #bytes()}) once done.
* <p>
* Please note, its is the responsibility of the caller to make sure the bytes
* reference do not "escape" and are released only once.
* Please note, closing this stream will release the bytes that are in use by any
* {@link ReleasablePagedBytesReference} returned from {@link #bytes()}, so this
* stream should only be closed after the bytes have been output or copied
* elsewhere.
*/
public class ReleasableBytesStreamOutput extends BytesStreamOutput implements ReleasableBytesStream {
public class ReleasableBytesStreamOutput extends BytesStreamOutput
implements Releasable {
private Releasable releasable;
public ReleasableBytesStreamOutput(BigArrays bigarrays) {
super(BigArrays.PAGE_SIZE_IN_BYTES, bigarrays);
this(BigArrays.PAGE_SIZE_IN_BYTES, bigarrays);
}
public ReleasableBytesStreamOutput(int expectedSize, BigArrays bigArrays) {
super(expectedSize, bigArrays);
this.releasable = Releasables.releaseOnce(this.bytes);
}
/**
* Returns a {@link Releasable} implementation of a
* {@link org.elasticsearch.common.bytes.BytesReference} that represents the current state of
* the bytes in the stream.
*/
@Override
public ReleasablePagedBytesReference bytes() {
return new ReleasablePagedBytesReference(bigArrays, bytes, count, releasable);
}
@Override
public ReleasablePagedBytesReference bytes() {
return new ReleasablePagedBytesReference(bigArrays, bytes, count);
public void close() {
Releasables.close(releasable);
}
@Override
void ensureCapacity(long offset) {
final ByteArray prevBytes = this.bytes;
super.ensureCapacity(offset);
if (prevBytes != this.bytes) {
// re-create the releasable with the new reference
releasable = Releasables.releaseOnce(this.bytes);
}
}
@Override
public void reset() {
final ByteArray prevBytes = this.bytes;
super.reset();
if (prevBytes != this.bytes) {
// re-create the releasable with the new reference
releasable = Releasables.releaseOnce(this.bytes);
}
}
}

33
core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
View File

@ -226,10 +226,41 @@ public class DeprecationLogger {
* @return the extracted warning value
*/
public static String extractWarningValueFromWarningHeader(final String s) {
/*
* We know the exact format of the warning header, so to extract the warning value we can skip forward from the front to the first
* quote, and skip backwards from the end to the penultimate quote:
*
* 299 Elasticsearch-6.0.0 "warning value" "Sat, 25, Feb 2017 10:27:43 GMT"
* ^ ^ ^
* firstQuote penultimateQuote lastQuote
*
* We do it this way rather than seeking forward after the first quote because there could be escaped quotes in the warning value
* but since there are none in the warning date, we can skip backwards to find the quote that closes the quoted warning value.
*
* We parse this manually rather than using the capturing regular expression because the regular expression involves a lot of
* backtracking and carries a performance penalty. However, when assertions are enabled, we still use the regular expression to
* verify that we are maintaining the warning header format.
*/
final int firstQuote = s.indexOf('\"');
final int lastQuote = s.lastIndexOf('\"');
final int penultimateQuote = s.lastIndexOf('\"', lastQuote - 1);
final String warningValue = s.substring(firstQuote + 1, penultimateQuote - 2);
assert assertWarningValue(s, warningValue);
return warningValue;
}
/**
* Assert that the specified string has the warning value equal to the provided warning value.
*
* @param s the string representing a full warning header
* @param warningValue the expected warning header
* @return {@code true} if the specified string has the expected warning value
*/
private static boolean assertWarningValue(final String s, final String warningValue) {
final Matcher matcher = WARNING_HEADER_PATTERN.matcher(s);
final boolean matches = matcher.matches();
assert matches;
return matcher.group(1);
return matcher.group(1).equals(warningValue);
}
/**

37
core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java → core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java
View File

@ -29,9 +29,12 @@ import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.uid.VersionsResolver.DocIdAndVersion;
import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo;
import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion;
import org.elasticsearch.index.mapper.SeqNoFieldMapper;
import org.elasticsearch.index.mapper.UidFieldMapper;
import org.elasticsearch.index.mapper.VersionFieldMapper;
import org.elasticsearch.index.seqno.SequenceNumbersService;
import java.io.IOException;
@ -43,7 +46,7 @@ import java.io.IOException;
* in more than one document! It will only return the first one it
* finds. */
final class PerThreadIDAndVersionLookup {
final class PerThreadIDVersionAndSeqNoLookup {
// TODO: do we really need to store all this stuff? some if it might not speed up anything.
// we keep it around for now, to reduce the amount of e.g. hash lookups by field and stuff
@ -51,7 +54,10 @@ final class PerThreadIDAndVersionLookup {
private final TermsEnum termsEnum;
/** _version data */
private final NumericDocValues versions;
/** _seq_no data */
private final NumericDocValues seqNos;
/** _primary_term data */
private final NumericDocValues primaryTerms;
/** Reused for iteration (when the term exists) */
private PostingsEnum docsEnum;
@ -61,7 +67,7 @@ final class PerThreadIDAndVersionLookup {
/**
* Initialize lookup for the provided segment
*/
PerThreadIDAndVersionLookup(LeafReader reader) throws IOException {
PerThreadIDVersionAndSeqNoLookup(LeafReader reader) throws IOException {
Fields fields = reader.fields();
Terms terms = fields.terms(UidFieldMapper.NAME);
termsEnum = terms.iterator();
@ -74,6 +80,8 @@ final class PerThreadIDAndVersionLookup {
throw new IllegalArgumentException("reader misses the [" + VersionFieldMapper.NAME +
"] field");
}
seqNos = reader.getNumericDocValues(SeqNoFieldMapper.NAME);
primaryTerms = reader.getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME);
Object readerKey = null;
assert (readerKey = reader.getCoreCacheKey()) != null;
this.readerKey = readerKey;
@ -113,4 +121,25 @@ final class PerThreadIDAndVersionLookup {
return DocIdSetIterator.NO_MORE_DOCS;
}
}
/** Return null if id is not found. */
DocIdAndSeqNo lookupSeqNo(BytesRef id, Bits liveDocs, LeafReaderContext context) throws IOException {
assert context.reader().getCoreCacheKey().equals(readerKey) :
"context's reader is not the same as the reader class was initialized on.";
int docID = getDocID(id, liveDocs);
if (docID != DocIdSetIterator.NO_MORE_DOCS) {
return new DocIdAndSeqNo(docID, seqNos == null ? SequenceNumbersService.UNASSIGNED_SEQ_NO : seqNos.get(docID), context);
} else {
return null;
}
}
/**
* returns 0 if the primary term is not found.
*
* Note that 0 is an illegal primary term. See {@link org.elasticsearch.cluster.metadata.IndexMetaData#primaryTerm(int)}
**/
long lookUpPrimaryTerm(int docID) throws IOException {
return primaryTerms == null ? 0 : primaryTerms.get(docID);
}
}

180
core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java Normal file
View File

@ -0,0 +1,180 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.lucene.uid;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReader.CoreClosedListener;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.CloseableThreadLocal;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.index.mapper.UidFieldMapper;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
import static org.elasticsearch.common.lucene.uid.Versions.NOT_FOUND;
/** Utility class to resolve the Lucene doc ID, version, seqNo and primaryTerms for a given uid. */
public final class VersionsAndSeqNoResolver {
static final ConcurrentMap<Object, CloseableThreadLocal<PerThreadIDVersionAndSeqNoLookup>> lookupStates =
ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
// Evict this reader from lookupStates once it's closed:
private static final CoreClosedListener removeLookupState = key -> {
CloseableThreadLocal<PerThreadIDVersionAndSeqNoLookup> ctl = lookupStates.remove(key);
if (ctl != null) {
ctl.close();
}
};
private static PerThreadIDVersionAndSeqNoLookup getLookupState(LeafReader reader) throws IOException {
Object key = reader.getCoreCacheKey();
CloseableThreadLocal<PerThreadIDVersionAndSeqNoLookup> ctl = lookupStates.get(key);
if (ctl == null) {
// First time we are seeing this reader's core; make a new CTL:
ctl = new CloseableThreadLocal<>();
CloseableThreadLocal<PerThreadIDVersionAndSeqNoLookup> other = lookupStates.putIfAbsent(key, ctl);
if (other == null) {
// Our CTL won, we must remove it when the core is closed:
reader.addCoreClosedListener(removeLookupState);
} else {
// Another thread beat us to it: just use their CTL:
ctl = other;
}
}
PerThreadIDVersionAndSeqNoLookup lookupState = ctl.get();
if (lookupState == null) {
lookupState = new PerThreadIDVersionAndSeqNoLookup(reader);
ctl.set(lookupState);
}
return lookupState;
}
private VersionsAndSeqNoResolver() {
}
/** Wraps an {@link LeafReaderContext}, a doc ID <b>relative to the context doc base</b> and a version. */
public static class DocIdAndVersion {
public final int docId;
public final long version;
public final LeafReaderContext context;
DocIdAndVersion(int docId, long version, LeafReaderContext context) {
this.docId = docId;
this.version = version;
this.context = context;
}
}
/** Wraps an {@link LeafReaderContext}, a doc ID <b>relative to the context doc base</b> and a seqNo. */
public static class DocIdAndSeqNo {
public final int docId;
public final long seqNo;
public final LeafReaderContext context;
DocIdAndSeqNo(int docId, long seqNo, LeafReaderContext context) {
this.docId = docId;
this.seqNo = seqNo;
this.context = context;
}
}
/**
* Load the internal doc ID and version for the uid from the reader, returning<ul>
* <li>null if the uid wasn't found,
* <li>a doc ID and a version otherwise
* </ul>
*/
public static DocIdAndVersion loadDocIdAndVersion(IndexReader reader, Term term) throws IOException {
assert term.field().equals(UidFieldMapper.NAME) : "unexpected term field " + term.field();
List<LeafReaderContext> leaves = reader.leaves();
if (leaves.isEmpty()) {
return null;
}
// iterate backwards to optimize for the frequently updated documents
// which are likely to be in the last segments
for (int i = leaves.size() - 1; i >= 0; i--) {
LeafReaderContext context = leaves.get(i);
LeafReader leaf = context.reader();
PerThreadIDVersionAndSeqNoLookup lookup = getLookupState(leaf);
DocIdAndVersion result = lookup.lookupVersion(term.bytes(), leaf.getLiveDocs(), context);
if (result != null) {
return result;
}
}
return null;
}
/**
* Load the internal doc ID and sequence number for the uid from the reader, returning<ul>
* <li>null if the uid wasn't found,
* <li>a doc ID and the associated seqNo otherwise
* </ul>
*/
public static DocIdAndSeqNo loadDocIdAndSeqNo(IndexReader reader, Term term) throws IOException {
assert term.field().equals(UidFieldMapper.NAME) : "unexpected term field " + term.field();
List<LeafReaderContext> leaves = reader.leaves();
if (leaves.isEmpty()) {
return null;
}
// iterate backwards to optimize for the frequently updated documents
// which are likely to be in the last segments
for (int i = leaves.size() - 1; i >= 0; i--) {
LeafReaderContext context = leaves.get(i);
LeafReader leaf = context.reader();
PerThreadIDVersionAndSeqNoLookup lookup = getLookupState(leaf);
DocIdAndSeqNo result = lookup.lookupSeqNo(term.bytes(), leaf.getLiveDocs(), context);
if (result != null) {
return result;
}
}
return null;
}
/**
* Load the primaryTerm associated with the given {@link DocIdAndSeqNo}
*/
public static long loadPrimaryTerm(DocIdAndSeqNo docIdAndSeqNo) throws IOException {
LeafReader leaf = docIdAndSeqNo.context.reader();
PerThreadIDVersionAndSeqNoLookup lookup = getLookupState(leaf);
long result = lookup.lookUpPrimaryTerm(docIdAndSeqNo.docId);
assert result > 0 : "should always resolve a primary term for a resolved sequence number. primary_term [" + result + "]"
+ " docId [" + docIdAndSeqNo.docId + "] seqNo [" + docIdAndSeqNo.seqNo + "]";
return result;
}
/**
* Load the version for the uid from the reader, returning<ul>
* <li>{@link Versions#NOT_FOUND} if no matching doc exists,
* <li>the version associated with the provided uid otherwise
* </ul>
*/
public static long loadVersion(IndexReader reader, Term term) throws IOException {
final DocIdAndVersion docIdAndVersion = loadDocIdAndVersion(reader, term);
return docIdAndVersion == null ? NOT_FOUND : docIdAndVersion.version;
}
}

263
core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsResolver.java
View File

@ -1,263 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.lucene.uid;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReader.CoreClosedListener;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CloseableThreadLocal;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.index.mapper.SeqNoFieldMapper;
import org.elasticsearch.index.mapper.UidFieldMapper;
import org.elasticsearch.index.seqno.SequenceNumbersService;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
import static org.elasticsearch.common.lucene.uid.Versions.NOT_FOUND;
/** Utility class to resolve the Lucene doc ID and version for a given uid. */
public class VersionsResolver {
static final ConcurrentMap<Object, CloseableThreadLocal<PerThreadIDAndVersionLookup>>
lookupStates = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
// Evict this reader from lookupStates once it's closed:
private static final CoreClosedListener removeLookupState = key -> {
CloseableThreadLocal<PerThreadIDAndVersionLookup> ctl = lookupStates.remove(key);
if (ctl != null) {
ctl.close();
}
};
private static PerThreadIDAndVersionLookup getLookupState(LeafReader reader)
throws IOException {
Object key = reader.getCoreCacheKey();
CloseableThreadLocal<PerThreadIDAndVersionLookup> ctl = lookupStates.get(key);
if (ctl == null) {
// First time we are seeing this reader's core; make a
// new CTL:
ctl = new CloseableThreadLocal<>();
CloseableThreadLocal<PerThreadIDAndVersionLookup> other =
lookupStates.putIfAbsent(key, ctl);
if (other == null) {
// Our CTL won, we must remove it when the
// core is closed:
reader.addCoreClosedListener(removeLookupState);
} else {
// Another thread beat us to it: just use
// their CTL:
ctl = other;
}
}
PerThreadIDAndVersionLookup lookupState = ctl.get();
if (lookupState == null) {
lookupState = new PerThreadIDAndVersionLookup(reader);
ctl.set(lookupState);
}
return lookupState;
}
private VersionsResolver() {
}
/**
* Wraps an {@link LeafReaderContext}, a doc ID <b>relative to the context doc base</b> and
* a version.
**/
public static class DocIdAndVersion {
public final int docId;
public final long version;
public final LeafReaderContext context;
public DocIdAndVersion(int docId, long version, LeafReaderContext context) {
this.docId = docId;
this.version = version;
this.context = context;
}
}
/**
* Load the internal doc ID and version for the uid from the reader, returning<ul>
* <li>null if the uid wasn't found,
* <li>a doc ID and a version otherwise
* </ul>
*/
public static DocIdAndVersion loadDocIdAndVersion(IndexReader reader, Term term)
throws IOException {
assert term.field().equals(UidFieldMapper.NAME);
List<LeafReaderContext> leaves = reader.leaves();
if (leaves.isEmpty()) {
return null;
}
// iterate backwards to optimize for the frequently updated documents
// which are likely to be in the last segments
for (int i = leaves.size() - 1; i >= 0; i--) {
LeafReaderContext context = leaves.get(i);
LeafReader leaf = context.reader();
PerThreadIDAndVersionLookup lookup = getLookupState(leaf);
DocIdAndVersion result =
lookup.lookupVersion(term.bytes(), leaf.getLiveDocs(), context);
if (result != null) {
return result;
}
}
return null;
}
/**
* Load the version for the uid from the reader, returning<ul>
* <li>{@link Versions#NOT_FOUND} if no matching doc exists,
* <li>the version associated with the provided uid otherwise
* </ul>
*/
public static long loadVersion(IndexReader reader, Term term) throws IOException {
final DocIdAndVersion docIdAndVersion = loadDocIdAndVersion(reader, term);
return docIdAndVersion == null ? NOT_FOUND : docIdAndVersion.version;
}
/**
* Returns the sequence number for the given uid term, returning
* {@code SequenceNumbersService.UNASSIGNED_SEQ_NO} if none is found.
*/
public static long loadSeqNo(IndexReader reader, Term term) throws IOException {
assert term.field().equals(UidFieldMapper.NAME) : "can only load _seq_no by uid";
List<LeafReaderContext> leaves = reader.leaves();
if (leaves.isEmpty()) {
return SequenceNumbersService.UNASSIGNED_SEQ_NO;
}
// iterate backwards to optimize for the frequently updated documents
// which are likely to be in the last segments
for (int i = leaves.size() - 1; i >= 0; i--) {
LeafReader leaf = leaves.get(i).reader();
Bits liveDocs = leaf.getLiveDocs();
TermsEnum termsEnum = null;
SortedNumericDocValues dvField = null;
PostingsEnum docsEnum = null;
final Fields fields = leaf.fields();
if (fields != null) {
Terms terms = fields.terms(UidFieldMapper.NAME);
if (terms != null) {
termsEnum = terms.iterator();
assert termsEnum != null;
dvField = leaf.getSortedNumericDocValues(SeqNoFieldMapper.NAME);
assert dvField != null;
final BytesRef id = term.bytes();
if (termsEnum.seekExact(id)) {
// there may be more than one matching docID, in the
// case of nested docs, so we want the last one:
docsEnum = termsEnum.postings(docsEnum, 0);
int docID = DocIdSetIterator.NO_MORE_DOCS;
for (int d = docsEnum.nextDoc();
d != DocIdSetIterator.NO_MORE_DOCS; d = docsEnum.nextDoc()) {
if (liveDocs != null && liveDocs.get(d) == false) {
continue;
}
docID = d;
}
if (docID != DocIdSetIterator.NO_MORE_DOCS) {
dvField.setDocument(docID);
assert dvField.count() == 1 :
"expected only a single value for _seq_no but got " +
dvField.count();
return dvField.valueAt(0);
}
}
}
}
}
return SequenceNumbersService.UNASSIGNED_SEQ_NO;
}
/**
* Returns the primary term for the given uid term, returning {@code 0} if none is found.
*/
public static long loadPrimaryTerm(IndexReader reader, Term term) throws IOException {
assert term.field().equals(UidFieldMapper.NAME) : "can only load _primary_term by uid";
List<LeafReaderContext> leaves = reader.leaves();
if (leaves.isEmpty()) {
return 0;
}
// iterate backwards to optimize for the frequently updated documents
// which are likely to be in the last segments
for (int i = leaves.size() - 1; i >= 0; i--) {
LeafReader leaf = leaves.get(i).reader();
Bits liveDocs = leaf.getLiveDocs();
TermsEnum termsEnum = null;
NumericDocValues dvField = null;
PostingsEnum docsEnum = null;
final Fields fields = leaf.fields();
if (fields != null) {
Terms terms = fields.terms(UidFieldMapper.NAME);
if (terms != null) {
termsEnum = terms.iterator();
assert termsEnum != null;
dvField = leaf.getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME);
assert dvField != null;
final BytesRef id = term.bytes();
if (termsEnum.seekExact(id)) {
// there may be more than one matching docID, in the
// case of nested docs, so we want the last one:
docsEnum = termsEnum.postings(docsEnum, 0);
int docID = DocIdSetIterator.NO_MORE_DOCS;
for (int d = docsEnum.nextDoc();
d != DocIdSetIterator.NO_MORE_DOCS;
d = docsEnum.nextDoc()) {
if (liveDocs != null && liveDocs.get(d) == false) {
continue;
}
docID = d;
}
if (docID != DocIdSetIterator.NO_MORE_DOCS) {
return dvField.get(docID);
}
}
}
}
}
return 0;
}
}

3
core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View File

@ -311,9 +311,12 @@ public final class ClusterSettings extends AbstractScopedSettings {
HunspellService.HUNSPELL_IGNORE_CASE,
HunspellService.HUNSPELL_DICTIONARY_OPTIONS,
IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT,
Environment.DEFAULT_PATH_CONF_SETTING,
Environment.PATH_CONF_SETTING,
Environment.DEFAULT_PATH_DATA_SETTING,
Environment.PATH_DATA_SETTING,
Environment.PATH_HOME_SETTING,
Environment.DEFAULT_PATH_LOGS_SETTING,
Environment.PATH_LOGS_SETTING,
Environment.PATH_REPO_SETTING,
Environment.PATH_SCRIPTS_SETTING,

21
core/src/main/java/org/elasticsearch/common/settings/Settings.java
View File

@ -57,6 +57,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
@ -442,6 +443,20 @@ public final class Settings implements ToXContent {
public String[] getAsArray(String settingPrefix, String[] defaultArray, Boolean commaDelimited) throws SettingsException {
List<String> result = new ArrayList<>();
final String valueFromPrefix = get(settingPrefix);
final String valueFromPreifx0 = get(settingPrefix + ".0");
if (valueFromPrefix != null && valueFromPreifx0 != null) {
final String message = String.format(
Locale.ROOT,
"settings object contains values for [%s=%s] and [%s=%s]",
settingPrefix,
valueFromPrefix,
settingPrefix + ".0",
valueFromPreifx0);
throw new IllegalStateException(message);
}
if (get(settingPrefix) != null) {
if (commaDelimited) {
String[] strings = Strings.splitStringByCommaToArray(get(settingPrefix));
@ -1048,12 +1063,10 @@ public final class Settings implements ToXContent {
return this;
}
public Builder putProperties(Map<String, String> esSettings, Predicate<String> keyPredicate, Function<String, String> keyFunction) {
public Builder putProperties(final Map<String, String> esSettings, final Function<String, String> keyFunction) {
for (final Map.Entry<String, String> esSetting : esSettings.entrySet()) {
final String key = esSetting.getKey();
if (keyPredicate.test(key)) {
map.put(keyFunction.apply(key), esSetting.getValue());
}
map.put(keyFunction.apply(key), esSetting.getValue());
}
return this;
}

5
core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
View File

@ -22,7 +22,7 @@ package org.elasticsearch.common.xcontent;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.io.BytesStream;
import org.elasticsearch.common.io.stream.BytesStream;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.text.Text;
@ -53,7 +53,7 @@ import java.util.concurrent.TimeUnit;
/**
* A utility to build XContent (ie json).
*/
public final class XContentBuilder implements BytesStream, Releasable, Flushable {
public final class XContentBuilder implements Releasable, Flushable {
/**
* Create a new {@link XContentBuilder} using the given {@link XContent} content.
@ -1041,7 +1041,6 @@ public final class XContentBuilder implements BytesStream, Releasable, Flushable
return this.generator;
}
@Override
public BytesReference bytes() {
close();
return ((BytesStream) bos).bytes();

15
core/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java
View File

@ -111,14 +111,14 @@ public class ElectMasterService extends AbstractComponent {
return minimumMasterNodes;
}
public boolean hasEnoughMasterNodes(Iterable<DiscoveryNode> nodes) {
public int countMasterNodes(Iterable<DiscoveryNode> nodes) {
int count = 0;
for (DiscoveryNode node : nodes) {
if (node.isMasterNode()) {
count++;
}
}
return count > 0 && (minimumMasterNodes < 0 || count >= minimumMasterNodes);
return count;
}
public boolean hasEnoughCandidates(Collection<MasterCandidate> candidates) {
@ -149,13 +149,12 @@ public class ElectMasterService extends AbstractComponent {
return activeMasters.stream().min(ElectMasterService::compareNodes).get();
}
public boolean hasEnoughMasterNodes(Iterable<DiscoveryNode> nodes) {
return minimumMasterNodes < 1 || countMasterNodes(nodes) >= minimumMasterNodes;
}
public boolean hasTooManyMasterNodes(Iterable<DiscoveryNode> nodes) {
int count = 0;
for (DiscoveryNode node : nodes) {
if (node.isMasterNode()) {
count++;
}
}
final int count = countMasterNodes(nodes);
return count > 1 && minimumMasterNodes <= count / 2;
}

10
core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
View File

@ -47,6 +47,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.logging.LoggerMessageFormat;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
@ -580,8 +581,10 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
final ClusterState remainingNodesClusterState = remainingNodesClusterState(currentState, remainingNodesBuilder);
final ClusterTasksResult.Builder<Task> resultBuilder = ClusterTasksResult.<Task>builder().successes(tasks);
if (!electMasterService.hasEnoughMasterNodes(remainingNodesClusterState.nodes())) {
rejoin.accept("not enough master nodes");
if (electMasterService.hasEnoughMasterNodes(remainingNodesClusterState.nodes()) == false) {
final int masterNodes = electMasterService.countMasterNodes(remainingNodesClusterState.nodes());
rejoin.accept(LoggerMessageFormat.format("not enough master nodes (has [{}], but needed [{}])",
masterNodes, electMasterService.minimumMasterNodes()));
return resultBuilder.build(currentState);
} else {
return resultBuilder.build(allocationService.deassociateDeadNodes(remainingNodesClusterState, true, describeTasks(tasks)));
@ -920,7 +923,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
return winner.getNode();
} else {
// if we don't have enough master nodes, we bail, because there are not enough master to elect from
logger.trace("not enough master nodes [{}]", masterCandidates);
logger.warn("not enough master nodes discovered during pinging (found [{}], but needed [{}]), pinging again",
masterCandidates, electMaster.minimumMasterNodes());
return null;
}
} else {

23
core/src/main/java/org/elasticsearch/env/Environment.java vendored
View File

@ -49,11 +49,17 @@ import static org.elasticsearch.common.Strings.cleanPath;
// public+forbidden api!
public class Environment {
public static final Setting<String> PATH_HOME_SETTING = Setting.simpleString("path.home", Property.NodeScope);
public static final Setting<String> PATH_CONF_SETTING = Setting.simpleString("path.conf", Property.NodeScope);
public static final Setting<String> DEFAULT_PATH_CONF_SETTING = Setting.simpleString("default.path.conf", Property.NodeScope);
public static final Setting<String> PATH_CONF_SETTING =
new Setting<>("path.conf", DEFAULT_PATH_CONF_SETTING, Function.identity(), Property.NodeScope);
public static final Setting<String> PATH_SCRIPTS_SETTING = Setting.simpleString("path.scripts", Property.NodeScope);
public static final Setting<List<String>> DEFAULT_PATH_DATA_SETTING =
Setting.listSetting("default.path.data", Collections.emptyList(), Function.identity(), Property.NodeScope);
public static final Setting<List<String>> PATH_DATA_SETTING =
Setting.listSetting("path.data", Collections.emptyList(), Function.identity(), Property.NodeScope);
public static final Setting<String> PATH_LOGS_SETTING = Setting.simpleString("path.logs", Property.NodeScope);
Setting.listSetting("path.data", DEFAULT_PATH_DATA_SETTING, Function.identity(), Property.NodeScope);
public static final Setting<String> DEFAULT_PATH_LOGS_SETTING = Setting.simpleString("default.path.logs", Property.NodeScope);
public static final Setting<String> PATH_LOGS_SETTING =
new Setting<>("path.logs", DEFAULT_PATH_LOGS_SETTING, Function.identity(), Property.NodeScope);
public static final Setting<List<String>> PATH_REPO_SETTING =
Setting.listSetting("path.repo", Collections.emptyList(), Function.identity(), Property.NodeScope);
public static final Setting<String> PATH_SHARED_DATA_SETTING = Setting.simpleString("path.shared_data", Property.NodeScope);
@ -115,7 +121,8 @@ public class Environment {
throw new IllegalStateException(PATH_HOME_SETTING.getKey() + " is not configured");
}
if (PATH_CONF_SETTING.exists(settings)) {
// this is trappy, Setting#get(Settings) will get a fallback setting yet return false for Settings#exists(Settings)
if (PATH_CONF_SETTING.exists(settings) || DEFAULT_PATH_CONF_SETTING.exists(settings)) {
configFile = PathUtils.get(cleanPath(PATH_CONF_SETTING.get(settings)));
} else {
configFile = homeFile.resolve("config");
@ -156,7 +163,9 @@ public class Environment {
} else {
repoFiles = new Path[0];
}
if (PATH_LOGS_SETTING.exists(settings)) {
// this is trappy, Setting#get(Settings) will get a fallback setting yet return false for Settings#exists(Settings)
if (PATH_LOGS_SETTING.exists(settings) || DEFAULT_PATH_LOGS_SETTING.exists(settings)) {
logsFile = PathUtils.get(cleanPath(PATH_LOGS_SETTING.get(settings)));
} else {
logsFile = homeFile.resolve("logs");
@ -174,7 +183,9 @@ public class Environment {
Settings.Builder finalSettings = Settings.builder().put(settings);
finalSettings.put(PATH_HOME_SETTING.getKey(), homeFile);
finalSettings.putArray(PATH_DATA_SETTING.getKey(), dataPaths);
if (PATH_DATA_SETTING.exists(settings)) {
finalSettings.putArray(PATH_DATA_SETTING.getKey(), dataPaths);
}
finalSettings.put(PATH_LOGS_SETTING.getKey(), logsFile);
this.settings = finalSettings.build();

56
core/src/main/java/org/elasticsearch/env/NodeEnvironment.java vendored
View File

@ -202,7 +202,7 @@ public final class NodeEnvironment implements Closeable {
for (int dirIndex = 0; dirIndex < environment.dataFiles().length; dirIndex++) {
Path dataDirWithClusterName = environment.dataWithClusterFiles()[dirIndex];
Path dataDir = environment.dataFiles()[dirIndex];
Path dir = dataDir.resolve(NODES_FOLDER).resolve(Integer.toString(possibleLockId));
Path dir = resolveNodePath(dataDir, possibleLockId);
Files.createDirectories(dir);
try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
@ -268,6 +268,17 @@ public final class NodeEnvironment implements Closeable {
}
}
/**
* Resolve a specific nodes/{node.id} path for the specified path and node lock id.
*
* @param path the path
* @param nodeLockId the node lock id
* @return the resolved path
*/
public static Path resolveNodePath(final Path path, final int nodeLockId) {
return path.resolve(NODES_FOLDER).resolve(Integer.toString(nodeLockId));
}
/** Returns true if the directory is empty */
private static boolean dirEmpty(final Path path) throws IOException {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
@ -724,6 +735,14 @@ public final class NodeEnvironment implements Closeable {
return nodePaths;
}
public int getNodeLockId() {
assertEnvIsLocked();
if (nodePaths == null || locks == null) {
throw new IllegalStateException("node is not configured to store local location");
}
return nodeLockId;
}
/**
* Returns all index paths.
*/
@ -736,6 +755,8 @@ public final class NodeEnvironment implements Closeable {
return indexPaths;
}
/**
* Returns all shard paths excluding custom shard path. Note: Shards are only allocated on one of the
* returned paths. The returned array may contain paths to non-existing directories.
@ -764,19 +785,36 @@ public final class NodeEnvironment implements Closeable {
assertEnvIsLocked();
Set<String> indexFolders = new HashSet<>();
for (NodePath nodePath : nodePaths) {
Path indicesLocation = nodePath.indicesPath;
if (Files.isDirectory(indicesLocation)) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(indicesLocation)) {
for (Path index : stream) {
if (Files.isDirectory(index)) {
indexFolders.add(index.getFileName().toString());
}
indexFolders.addAll(availableIndexFoldersForPath(nodePath));
}
return indexFolders;
}
/**
* Return all directory names in the nodes/{node.id}/indices directory for the given node path.
*
* @param nodePath the path
* @return all directories that could be indices for the given node path.
* @throws IOException if an I/O exception occurs traversing the filesystem
*/
public Set<String> availableIndexFoldersForPath(final NodePath nodePath) throws IOException {
if (nodePaths == null || locks == null) {
throw new IllegalStateException("node is not configured to store local location");
}
assertEnvIsLocked();
final Set<String> indexFolders = new HashSet<>();
Path indicesLocation = nodePath.indicesPath;
if (Files.isDirectory(indicesLocation)) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(indicesLocation)) {
for (Path index : stream) {
if (Files.isDirectory(index)) {
indexFolders.add(index.getFileName().toString());
}
}
}
}
return indexFolders;
}
/**

15
core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java
View File

@ -27,18 +27,13 @@ class DeleteVersionValue extends VersionValue {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DeleteVersionValue.class);
private final long time;
final long time;
DeleteVersionValue(long version, long time) {
super(version);
DeleteVersionValue(long version,long seqNo, long term, long time) {
super(version, seqNo, term);
this.time = time;
}
@Override
public long getTime() {
return this.time;
}
@Override
public boolean isDelete() {
return true;
@ -52,7 +47,9 @@ class DeleteVersionValue extends VersionValue {
@Override
public String toString() {
return "DeleteVersionValue{" +
"version=" + getVersion() +
"version=" + version +
", seqNo=" + seqNo +
", term=" + term +
",time=" + time +
'}';
}

6
core/src/main/java/org/elasticsearch/index/engine/Engine.java
View File

@ -55,8 +55,8 @@ import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.uid.Versions;
import org.elasticsearch.common.lucene.uid.VersionsResolver;
import org.elasticsearch.common.lucene.uid.VersionsResolver.DocIdAndVersion;
import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver;
import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.ReleasableLock;
@ -464,7 +464,7 @@ public abstract class Engine implements Closeable {
final Searcher searcher = searcherFactory.apply("get");
final DocIdAndVersion docIdAndVersion;
try {
docIdAndVersion = VersionsResolver.loadDocIdAndVersion(searcher.reader(), get.uid());
docIdAndVersion = VersionsAndSeqNoResolver.loadDocIdAndVersion(searcher.reader(), get.uid());
} catch (Exception e) {
Releasables.closeWhileHandlingException(searcher);
//TODO: A better exception goes here

154
core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
View File

@ -51,7 +51,8 @@ import org.elasticsearch.common.lucene.LoggerInfoStream;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
import org.elasticsearch.common.lucene.uid.Versions;
import org.elasticsearch.common.lucene.uid.VersionsResolver;
import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver;
import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
@ -298,7 +299,7 @@ public class InternalEngine extends Engine {
throw new IllegalStateException("no translog generation present in commit data but translog is expected to exist");
}
if (generation.translogUUID == null) {
throw new IndexFormatTooOldException("trasnlog", "translog has no generation nor a UUID - this might be an index from a previous version consider upgrading to N-1 first");
throw new IndexFormatTooOldException("translog", "translog has no generation nor a UUID - this might be an index from a previous version consider upgrading to N-1 first");
}
}
final Translog translog = new Translog(translogConfig, generation, globalCheckpointSupplier);
@ -389,10 +390,10 @@ public class InternalEngine extends Engine {
if (versionValue.isDelete()) {
return GetResult.NOT_EXISTS;
}
if (get.versionType().isVersionConflictForReads(versionValue.getVersion(), get.version())) {
if (get.versionType().isVersionConflictForReads(versionValue.version, get.version())) {
Uid uid = Uid.createUid(get.uid().text());
throw new VersionConflictEngineException(shardId, uid.type(), uid.id(),
get.versionType().explainConflictForReads(versionValue.getVersion(), get.version()));
get.versionType().explainConflictForReads(versionValue.version, get.version()));
}
refresh("realtime_get");
}
@ -416,6 +417,43 @@ public class InternalEngine extends Engine {
LUCENE_DOC_NOT_FOUND
}
private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnSeqNo(final Operation op) throws IOException {
assert op.seqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO : "resolving ops based on seq# but no seqNo is found";
final OpVsLuceneDocStatus status;
final VersionValue versionValue = versionMap.getUnderLock(op.uid());
assert incrementVersionLookup();
if (versionValue != null) {
if (op.seqNo() > versionValue.seqNo ||
(op.seqNo() == versionValue.seqNo && op.primaryTerm() > versionValue.term))
status = OpVsLuceneDocStatus.OP_NEWER;
else {
status = OpVsLuceneDocStatus.OP_STALE_OR_EQUAL;
}
} else {
// load from index
assert incrementIndexVersionLookup();
try (Searcher searcher = acquireSearcher("load_seq_no")) {
DocIdAndSeqNo docAndSeqNo = VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.reader(), op.uid());
if (docAndSeqNo == null) {
status = OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND;
} else if (op.seqNo() > docAndSeqNo.seqNo) {
status = OpVsLuceneDocStatus.OP_NEWER;
} else if (op.seqNo() == docAndSeqNo.seqNo) {
// load term to tie break
final long existingTerm = VersionsAndSeqNoResolver.loadPrimaryTerm(docAndSeqNo);
if (op.primaryTerm() > existingTerm) {
status = OpVsLuceneDocStatus.OP_NEWER;
} else {
status = OpVsLuceneDocStatus.OP_STALE_OR_EQUAL;
}
} else {
status = OpVsLuceneDocStatus.OP_STALE_OR_EQUAL;
}
}
}
return status;
}
/** resolves the current version of the document, returning null if not found */
private VersionValue resolveDocVersion(final Operation op) throws IOException {
assert incrementVersionLookup(); // used for asserting in tests
@ -424,11 +462,10 @@ public class InternalEngine extends Engine {
assert incrementIndexVersionLookup(); // used for asserting in tests
final long currentVersion = loadCurrentVersionFromIndex(op.uid());
if (currentVersion != Versions.NOT_FOUND) {
versionValue = new VersionValue(currentVersion);
versionValue = new VersionValue(currentVersion, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0L);
}
} else if (engineConfig.isEnableGcDeletes() && versionValue.isDelete() &&
(engineConfig.getThreadPool().relativeTimeInMillis() - versionValue.getTime()) >
getGcDeletesInMillis()) {
(engineConfig.getThreadPool().relativeTimeInMillis() - ((DeleteVersionValue)versionValue).time) > getGcDeletesInMillis()) {
versionValue = null;
}
return versionValue;
@ -436,12 +473,13 @@ public class InternalEngine extends Engine {
private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnVersions(final Operation op)
throws IOException {
assert op.seqNo() == SequenceNumbersService.UNASSIGNED_SEQ_NO : "op is resolved based on versions but have a seq#";
assert op.version() >= 0 : "versions should be non-negative. got " + op.version();
final VersionValue versionValue = resolveDocVersion(op);
if (versionValue == null) {
return OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND;
} else {
return op.versionType().isVersionConflictForWrites(versionValue.getVersion(), op.version(), versionValue.isDelete()) ?
return op.versionType().isVersionConflictForWrites(versionValue.version, op.version(), versionValue.isDelete()) ?
OpVsLuceneDocStatus.OP_STALE_OR_EQUAL : OpVsLuceneDocStatus.OP_NEWER;
}
}
@ -601,7 +639,16 @@ public class InternalEngine extends Engine {
// unlike the primary, replicas don't really care to about creation status of documents
// this allows to ignore the case where a document was found in the live version maps in
// a delete state and return false for the created flag in favor of code simplicity
final OpVsLuceneDocStatus opVsLucene = compareOpToLuceneDocBasedOnVersions(index);
final OpVsLuceneDocStatus opVsLucene;
if (index.seqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO) {
opVsLucene = compareOpToLuceneDocBasedOnSeqNo(index);
} else {
// This can happen if the primary is still on an old node and send traffic without seq# or we recover from translog
// created by an old version.
assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1_UNRELEASED) :
"index is newly created but op has no sequence numbers. op: " + index;
opVsLucene = compareOpToLuceneDocBasedOnVersions(index);
}
if (opVsLucene == OpVsLuceneDocStatus.OP_STALE_OR_EQUAL) {
plan = IndexingStrategy.processButSkipLucene(false, index.seqNo(), index.version());
} else {
@ -633,7 +680,7 @@ public class InternalEngine extends Engine {
currentVersion = Versions.NOT_FOUND;
currentNotFoundOrDeleted = true;
} else {
currentVersion = versionValue.getVersion();
currentVersion = versionValue.version;
currentNotFoundOrDeleted = versionValue.isDelete();
}
if (index.versionType().isVersionConflictForWrites(
@ -671,9 +718,9 @@ public class InternalEngine extends Engine {
assert assertDocDoesNotExist(index, canOptimizeAddDocument(index) == false);
index(index.docs(), indexWriter);
}
versionMap.putUnderLock(index.uid().bytes(), new VersionValue(plan.versionForIndexing));
return new IndexResult(plan.versionForIndexing, plan.seqNoForIndexing,
plan.currentNotFoundOrDeleted);
versionMap.putUnderLock(index.uid().bytes(),
new VersionValue(plan.versionForIndexing, plan.seqNoForIndexing, index.primaryTerm()));
return new IndexResult(plan.versionForIndexing, plan.seqNoForIndexing, plan.currentNotFoundOrDeleted);
} catch (Exception ex) {
if (indexWriter.getTragicException() == null) {
/* There is no tragic event recorded so this must be a document failure.
@ -873,7 +920,14 @@ public class InternalEngine extends Engine {
// unlike the primary, replicas don't really care to about found status of documents
// this allows to ignore the case where a document was found in the live version maps in
// a delete state and return true for the found flag in favor of code simplicity
final OpVsLuceneDocStatus opVsLucene = compareOpToLuceneDocBasedOnVersions(delete);
final OpVsLuceneDocStatus opVsLucene;
if (delete.seqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO) {
opVsLucene = compareOpToLuceneDocBasedOnSeqNo(delete);
} else {
assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1_UNRELEASED) :
"index is newly created but op has no sequence numbers. op: " + delete;
opVsLucene = compareOpToLuceneDocBasedOnVersions(delete);
}
final DeletionStrategy plan;
if (opVsLucene == OpVsLuceneDocStatus.OP_STALE_OR_EQUAL) {
@ -898,7 +952,7 @@ public class InternalEngine extends Engine {
currentVersion = Versions.NOT_FOUND;
currentlyDeleted = true;
} else {
currentVersion = versionValue.getVersion();
currentVersion = versionValue.version;
currentlyDeleted = versionValue.isDelete();
}
final DeletionStrategy plan;
@ -923,7 +977,7 @@ public class InternalEngine extends Engine {
indexWriter.deleteDocuments(delete.uid());
}
versionMap.putUnderLock(delete.uid().bytes(),
new DeleteVersionValue(plan.versionOfDeletion,
new DeleteVersionValue(plan.versionOfDeletion, plan.seqNoOfDeletion, delete.primaryTerm(),
engineConfig.getThreadPool().relativeTimeInMillis()));
return new DeleteResult(
plan.versionOfDeletion, plan.seqNoOfDeletion, plan.currentlyDeleted == false);
@ -1179,12 +1233,12 @@ public class InternalEngine extends Engine {
try {
translog.prepareCommit();
logger.trace("starting commit for flush; commitTranslog=true");
commitIndexWriter(indexWriter, translog, null);
final long committedGeneration = commitIndexWriter(indexWriter, translog, null);
logger.trace("finished commit for flush");
// we need to refresh in order to clear older version values
refresh("version_table_flush");
// after refresh documents can be retrieved from the index so we can now commit the translog
translog.commit();
translog.commit(committedGeneration);
} catch (Exception e) {
throw new FlushFailedEngineException(shardId, e);
}
@ -1235,14 +1289,14 @@ public class InternalEngine extends Engine {
// TODO: not good that we reach into LiveVersionMap here; can we move this inside VersionMap instead? problem is the dirtyLock...
// we only need to prune the deletes map; the current/old version maps are cleared on refresh:
for (Map.Entry<BytesRef, VersionValue> entry : versionMap.getAllTombstones()) {
for (Map.Entry<BytesRef, DeleteVersionValue> entry : versionMap.getAllTombstones()) {
BytesRef uid = entry.getKey();
try (Releasable ignored = acquireLock(uid)) { // can we do it without this lock on each value? maybe batch to a set and get the lock once per set?
// Must re-get it here, vs using entry.getValue(), in case the uid was indexed/deleted since we pulled the iterator:
VersionValue versionValue = versionMap.getTombstoneUnderLock(uid);
DeleteVersionValue versionValue = versionMap.getTombstoneUnderLock(uid);
if (versionValue != null) {
if (timeMSec - versionValue.getTime() > getGcDeletesInMillis()) {
if (timeMSec - versionValue.time > getGcDeletesInMillis()) {
versionMap.removeTombstoneUnderLock(uid);
}
}
@ -1490,7 +1544,7 @@ public class InternalEngine extends Engine {
private long loadCurrentVersionFromIndex(Term uid) throws IOException {
assert incrementIndexVersionLookup();
try (Searcher searcher = acquireSearcher("load_version")) {
return VersionsResolver.loadVersion(searcher.reader(), uid);
return VersionsAndSeqNoResolver.loadVersion(searcher.reader(), uid);
}
}
@ -1680,55 +1734,65 @@ public class InternalEngine extends Engine {
}
}
private void commitIndexWriter(IndexWriter writer, Translog translog, String syncId) throws IOException {
/**
* Commits the specified index writer.
*
* @param writer the index writer to commit
* @param translog the translog
* @param syncId the sync flush ID ({@code null} if not committing a synced flush)
* @return the minimum translog generation for the local checkpoint committed with the specified index writer
* @throws IOException if an I/O exception occurs committing the specfied writer
*/
private long commitIndexWriter(final IndexWriter writer, final Translog translog, @Nullable final String syncId) throws IOException {
ensureCanFlush();
try {
Translog.TranslogGeneration translogGeneration = translog.getGeneration();
final String translogFileGen = Long.toString(translogGeneration.translogFileGeneration);
final long localCheckpoint = seqNoService().getLocalCheckpoint();
final Translog.TranslogGeneration translogGeneration = translog.getMinGenerationForSeqNo(localCheckpoint + 1);
final String translogFileGeneration = Long.toString(translogGeneration.translogFileGeneration);
final String translogUUID = translogGeneration.translogUUID;
final String localCheckpoint = Long.toString(seqNoService().getLocalCheckpoint());
final String localCheckpointValue = Long.toString(localCheckpoint);
writer.setLiveCommitData(() -> {
/*
* The user data captured above (e.g. local checkpoint) contains data that must be evaluated *before* Lucene flushes
* segments, including the local checkpoint amongst other values. The maximum sequence number is different - we never want
* segments, including the local checkpoint amongst other values. The maximum sequence number is different, we never want
* the maximum sequence number to be less than the last sequence number to go into a Lucene commit, otherwise we run the
* risk of re-using a sequence number for two different documents when restoring from this commit point and subsequently
* writing new documents to the index. Since we only know which Lucene documents made it into the final commit after the
* {@link IndexWriter#commit()} call flushes all documents, we defer computation of the max_seq_no to the time of invocation
* of the commit data iterator (which occurs after all documents have been flushed to Lucene).
* writing new documents to the index. Since we only know which Lucene documents made it into the final commit after the
* {@link IndexWriter#commit()} call flushes all documents, we defer computation of the maximum sequence number to the time
* of invocation of the commit data iterator (which occurs after all documents have been flushed to Lucene).
*/
final Map<String, String> commitData = new HashMap<>(6);
commitData.put(Translog.TRANSLOG_GENERATION_KEY, translogFileGen);
final Map<String, String> commitData = new HashMap<>(5);
commitData.put(Translog.TRANSLOG_GENERATION_KEY, translogFileGeneration);
commitData.put(Translog.TRANSLOG_UUID_KEY, translogUUID);
commitData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, localCheckpoint);
commitData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, localCheckpointValue);
if (syncId != null) {
commitData.put(Engine.SYNC_COMMIT_ID, syncId);
}
commitData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(seqNoService().getMaxSeqNo()));
if (logger.isTraceEnabled()) {
logger.trace("committing writer with commit data [{}]", commitData);
}
logger.trace("committing writer with commit data [{}]", commitData);
return commitData.entrySet().iterator();
});
writer.commit();
} catch (Exception ex) {
return translogGeneration.translogFileGeneration;
} catch (final Exception ex) {
try {
failEngine("lucene commit failed", ex);
} catch (Exception inner) {
} catch (final Exception inner) {
ex.addSuppressed(inner);
}
throw ex;
} catch (AssertionError e) {
// IndexWriter throws AssertionError on commit, if asserts are enabled, if any files don't exist, but tests that
// randomly throw FNFE/NSFE can also hit this:
} catch (final AssertionError e) {
/*
* If assertions are enabled, IndexWriter throws AssertionError on commit if any files don't exist, but tests that randomly
* throw FileNotFoundException or NoSuchFileException can also hit this.
*/
if (ExceptionsHelper.stackTrace(e).contains("org.apache.lucene.index.IndexWriter.filesExist")) {
EngineException engineException = new EngineException(shardId, "failed to commit engine", e);
final EngineException engineException = new EngineException(shardId, "failed to commit engine", e);
try {
failEngine("lucene commit failed", engineException);
} catch (Exception inner) {
} catch (final Exception inner) {
engineException.addSuppressed(inner);
}
throw engineException;
@ -1812,7 +1876,7 @@ public class InternalEngine extends Engine {
* Gets the commit data from {@link IndexWriter} as a map.
*/
private static Map<String, String> commitDataAsMap(final IndexWriter indexWriter) {
Map<String, String> commitData = new HashMap<>(6);
Map<String, String> commitData = new HashMap<>(5);
for (Map.Entry<String, String> entry : indexWriter.getLiveCommitData()) {
commitData.put(entry.getKey(), entry.getValue());
}

8
core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
View File

@ -55,7 +55,7 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
}
// All deletes also go here, and delete "tombstones" are retained after refresh:
private final Map<BytesRef,VersionValue> tombstones = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
private final Map<BytesRef,DeleteVersionValue> tombstones = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
private volatile Maps maps = new Maps();
@ -180,7 +180,7 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
final VersionValue prevTombstone;
if (version.isDelete()) {
// Also enroll the delete into tombstones, and account for its RAM too:
prevTombstone = tombstones.put(uid, version);
prevTombstone = tombstones.put(uid, (DeleteVersionValue)version);
// We initially account for BytesRef/VersionValue RAM for a delete against the tombstones, because this RAM will not be freed up
// on refresh. Later, in removeTombstoneUnderLock, if we clear the tombstone entry but the delete remains in current, we shift
@ -225,12 +225,12 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
}
/** Caller has a lock, so that this uid will not be concurrently added/deleted by another thread. */
VersionValue getTombstoneUnderLock(BytesRef uid) {
DeleteVersionValue getTombstoneUnderLock(BytesRef uid) {
return tombstones.get(uid);
}
/** Iterates over all deleted versions, including new ones (not yet exposed via reader) and old ones (exposed via reader but not yet GC'd). */
Iterable<Map.Entry<BytesRef,VersionValue>> getAllTombstones() {
Iterable<Map.Entry<BytesRef, DeleteVersionValue>> getAllTombstones() {
return tombstones.entrySet();
}

24
core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
View File

@ -30,18 +30,17 @@ class VersionValue implements Accountable {
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(VersionValue.class);
/** the version of the document. used for versioned indexed operations and as a BWC layer, where no seq# are set yet */
private final long version;
final long version;
VersionValue(long version) {
/** the seq number of the operation that last changed the associated uuid */
final long seqNo;
/** the the term of the operation that last changed the associated uuid */
final long term;
VersionValue(long version, long seqNo, long term) {
this.version = version;
}
public long getTime() {
throw new UnsupportedOperationException();
}
public long getVersion() {
return version;
this.seqNo = seqNo;
this.term = term;
}
public boolean isDelete() {
@ -61,6 +60,9 @@ class VersionValue implements Accountable {
@Override
public String toString() {
return "VersionValue{" +
"version=" + version + "}";
"version=" + version +
", seqNo=" + seqNo +
", term=" + term +
'}';
}
}

2
core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
View File

@ -24,7 +24,7 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.lucene.uid.VersionsResolver.DocIdAndVersion;
import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.util.set.Sets;

14
core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
View File

@ -254,12 +254,12 @@ public abstract class ParseContext {
}
@Override
public SeqNoFieldMapper.SequenceID seqID() {
public SeqNoFieldMapper.SequenceIDFields seqID() {
return in.seqID();
}
@Override
public void seqID(SeqNoFieldMapper.SequenceID seqID) {
public void seqID(SeqNoFieldMapper.SequenceIDFields seqID) {
in.seqID(seqID);
}
@ -310,7 +310,7 @@ public abstract class ParseContext {
private Field version;
private SeqNoFieldMapper.SequenceID seqID;
private SeqNoFieldMapper.SequenceIDFields seqID;
private final AllEntries allEntries;
@ -404,12 +404,12 @@ public abstract class ParseContext {
}
@Override
public SeqNoFieldMapper.SequenceID seqID() {
public SeqNoFieldMapper.SequenceIDFields seqID() {
return this.seqID;
}
@Override
public void seqID(SeqNoFieldMapper.SequenceID seqID) {
public void seqID(SeqNoFieldMapper.SequenceIDFields seqID) {
this.seqID = seqID;
}
@ -539,9 +539,9 @@ public abstract class ParseContext {
public abstract void version(Field version);
public abstract SeqNoFieldMapper.SequenceID seqID();
public abstract SeqNoFieldMapper.SequenceIDFields seqID();
public abstract void seqID(SeqNoFieldMapper.SequenceID seqID);
public abstract void seqID(SeqNoFieldMapper.SequenceIDFields seqID);
public final boolean includeInAll(Boolean includeInAll, FieldMapper mapper) {
return includeInAll(includeInAll, mapper.fieldType().indexOptions() != IndexOptions.NONE);

4
core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
View File

@ -36,7 +36,7 @@ public class ParsedDocument {
private final String id, type;
private final BytesRef uid;
private final SeqNoFieldMapper.SequenceID seqID;
private final SeqNoFieldMapper.SequenceIDFields seqID;
private final String routing;
@ -50,7 +50,7 @@ public class ParsedDocument {
private String parent;
public ParsedDocument(Field version,
SeqNoFieldMapper.SequenceID seqID,
SeqNoFieldMapper.SequenceIDFields seqID,
String id,
String type,
String routing,

15
core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java
View File

@ -22,7 +22,6 @@ package org.elasticsearch.index.mapper;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
@ -66,13 +65,13 @@ public class SeqNoFieldMapper extends MetadataFieldMapper {
* A sequence ID, which is made up of a sequence number (both the searchable
* and doc_value version of the field) and the primary term.
*/
public static class SequenceID {
public static class SequenceIDFields {
public final Field seqNo;
public final Field seqNoDocValue;
public final Field primaryTerm;
public SequenceID(Field seqNo, Field seqNoDocValue, Field primaryTerm) {
public SequenceIDFields(Field seqNo, Field seqNoDocValue, Field primaryTerm) {
Objects.requireNonNull(seqNo, "sequence number field cannot be null");
Objects.requireNonNull(seqNoDocValue, "sequence number dv field cannot be null");
Objects.requireNonNull(primaryTerm, "primary term field cannot be null");
@ -81,9 +80,9 @@ public class SeqNoFieldMapper extends MetadataFieldMapper {
this.primaryTerm = primaryTerm;
}
public static SequenceID emptySeqID() {
return new SequenceID(new LongPoint(NAME, SequenceNumbersService.UNASSIGNED_SEQ_NO),
new SortedNumericDocValuesField(NAME, SequenceNumbersService.UNASSIGNED_SEQ_NO),
public static SequenceIDFields emptySeqID() {
return new SequenceIDFields(new LongPoint(NAME, SequenceNumbersService.UNASSIGNED_SEQ_NO),
new NumericDocValuesField(NAME, SequenceNumbersService.UNASSIGNED_SEQ_NO),
new NumericDocValuesField(PRIMARY_TERM_NAME, 0));
}
}
@ -242,7 +241,7 @@ public class SeqNoFieldMapper extends MetadataFieldMapper {
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
// see InternalEngine.innerIndex to see where the real version value is set
// also see ParsedDocument.updateSeqID (called by innerIndex)
SequenceID seqID = SequenceID.emptySeqID();
SequenceIDFields seqID = SequenceIDFields.emptySeqID();
context.seqID(seqID);
fields.add(seqID.seqNo);
fields.add(seqID.seqNoDocValue);
@ -264,7 +263,7 @@ public class SeqNoFieldMapper extends MetadataFieldMapper {
for (int i = 1; i < context.docs().size(); i++) {
final Document doc = context.docs().get(i);
doc.add(new LongPoint(NAME, 1));
doc.add(new SortedNumericDocValuesField(NAME, 1L));
doc.add(new NumericDocValuesField(NAME, 1L));
doc.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0L));
}
}

2
core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
View File

@ -34,7 +34,7 @@ import org.elasticsearch.action.termvectors.TermVectorsResponse;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.lucene.uid.VersionsResolver.DocIdAndVersion;
import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.support.XContentMapValues;

107
core/src/main/java/org/elasticsearch/index/translog/Translog.java
View File

@ -85,14 +85,14 @@ import java.util.stream.Stream;
* When a translog is opened the checkpoint is use to retrieve the latest translog file generation and subsequently to open the last written file to recovery operations.
* The {@link org.elasticsearch.index.translog.Translog.TranslogGeneration}, given when the translog is opened / constructed is compared against
* the latest generation and all consecutive translog files singe the given generation and the last generation in the checkpoint will be recovered and preserved until the next
* generation is committed using {@link Translog#commit()}. In the common case the translog file generation in the checkpoint and the generation passed to the translog on creation are
* the same. The only situation when they can be different is when an actual translog commit fails in between {@link Translog#prepareCommit()} and {@link Translog#commit()}. In such a case
* generation is committed using {@link Translog#commit(long)}. In the common case the translog file generation in the checkpoint and the generation passed to the translog on creation are
* the same. The only situation when they can be different is when an actual translog commit fails in between {@link Translog#prepareCommit()} and {@link Translog#commit(long)}. In such a case
* the currently being committed translog file will not be deleted since it's commit was not successful. Yet, a new/current translog file is already opened at that point such that there is more than
* one translog file present. Such an uncommitted translog file always has a <tt>translog-${gen}.ckp</tt> associated with it which is an fsynced copy of the it's last <tt>translog.ckp</tt> such that in
* disaster recovery last fsynced offsets, number of operation etc. are still preserved.
* </p>
*/
public class Translog extends AbstractIndexShardComponent implements IndexShardComponent, Closeable, TwoPhaseCommit {
public class Translog extends AbstractIndexShardComponent implements IndexShardComponent, Closeable {
/*
* TODO
@ -439,7 +439,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
}
throw new TranslogException(shardId, "Failed to write operation [" + operation + "]", e);
} finally {
Releasables.close(out.bytes());
Releasables.close(out);
}
}
@ -804,6 +804,8 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
long seqNo();
long primaryTerm();
/**
* Reads the type and the operation from the given stream. The operation must be written with
* {@link Operation#writeType(Operation, StreamOutput)}
@ -953,6 +955,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
return seqNo;
}
@Override
public long primaryTerm() {
return primaryTerm;
}
@ -1104,6 +1107,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
return seqNo;
}
@Override
public long primaryTerm() {
return primaryTerm;
}
@ -1180,6 +1184,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
return seqNo;
}
@Override
public long primaryTerm() {
return primaryTerm;
}
@ -1332,7 +1337,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
bytes.writeTo(outStream);
}
} finally {
Releasables.close(out.bytes());
Releasables.close(out);
}
}
@ -1347,6 +1352,31 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
out.writeInt((int) checksum);
}
/**
* Gets the minimum generation that could contain any sequence number after the specified sequence number, or the current generation if
* there is no generation that could any such sequence number.
*
* @param seqNo the sequence number
* @return the minimum generation for the sequence number
*/
public TranslogGeneration getMinGenerationForSeqNo(final long seqNo) {
try (ReleasableLock ignored = writeLock.acquire()) {
/*
* When flushing, the engine will ask the translog for the minimum generation that could contain any sequence number after the
* local checkpoint. Immediately after flushing, there will be no such generation, so this minimum generation in this case will
* be the current translog generation as we do not need any prior generations to have a complete history up to the current local
* checkpoint.
*/
long minTranslogFileGeneration = this.currentFileGeneration();
for (final TranslogReader reader : readers) {
if (seqNo <= reader.getCheckpoint().maxSeqNo) {
minTranslogFileGeneration = Math.min(minTranslogFileGeneration, reader.getGeneration());
}
}
return new TranslogGeneration(translogUUID, minTranslogFileGeneration);
}
}
/**
* Roll the current translog generation into a new generation. This does not commit the
* translog.
@ -1375,27 +1405,38 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
}
}
@Override
public long prepareCommit() throws IOException {
/**
* Prepares a translog commit by setting the current committing generation and rolling the translog generation.
*
* @throws IOException if an I/O exception occurred while rolling the translog generation
*/
public void prepareCommit() throws IOException {
try (ReleasableLock ignored = writeLock.acquire()) {
ensureOpen();
if (currentCommittingGeneration != NOT_SET_GENERATION) {
final String message = String.format(
Locale.ROOT,
"already committing a translog with generation [%d]",
currentCommittingGeneration);
final String message =
String.format(Locale.ROOT, "already committing a translog with generation [%d]", currentCommittingGeneration);
throw new IllegalStateException(message);
}
currentCommittingGeneration = current.getGeneration();
rollGeneration();
}
return 0;
}
@Override
public long commit() throws IOException {
/**
* Commits the translog and sets the last committed translog generation to the specified generation. The specified committed generation
* will be used when trimming unreferenced translog generations such that generations from the committed generation will be preserved.
*
* If {@link Translog#prepareCommit()} was not called before calling commit, this method will be invoked too causing the translog
* generation to be rolled.
*
* @param committedGeneration the minimum translog generation to preserve after trimming unreferenced generations
* @throws IOException if an I/O exception occurred preparing the translog commit
*/
public void commit(final long committedGeneration) throws IOException {
try (ReleasableLock ignored = writeLock.acquire()) {
ensureOpen();
assert assertCommittedGenerationIsInValidRange(committedGeneration);
if (currentCommittingGeneration == NOT_SET_GENERATION) {
prepareCommit();
}
@ -1403,26 +1444,39 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
assert readers.stream().anyMatch(r -> r.getGeneration() == currentCommittingGeneration)
: "readers missing committing generation [" + currentCommittingGeneration + "]";
// set the last committed generation otherwise old files will not be cleaned up
lastCommittedTranslogFileGeneration = currentCommittingGeneration + 1;
lastCommittedTranslogFileGeneration = committedGeneration;
currentCommittingGeneration = NOT_SET_GENERATION;
trimUnreferencedReaders();
}
return 0;
}
private boolean assertCommittedGenerationIsInValidRange(final long committedGeneration) {
assert committedGeneration <= current.generation
: "tried to commit generation [" + committedGeneration + "] after current generation [" + current.generation + "]";
final long min = readers.stream().map(TranslogReader::getGeneration).min(Long::compareTo).orElse(Long.MIN_VALUE);
assert committedGeneration >= min
: "tried to commit generation [" + committedGeneration + "] before minimum generation [" + min + "]";
return true;
}
/**
* Trims unreferenced translog generations. The guarantee here is that translog generations will be preserved for all outstanding views
* and from the last committed translog generation defined by {@link Translog#lastCommittedTranslogFileGeneration}.
*/
void trimUnreferencedReaders() {
try (ReleasableLock ignored = writeLock.acquire()) {
if (closed.get()) {
// we're shutdown potentially on some tragic event - don't delete anything
// we're shutdown potentially on some tragic event, don't delete anything
return;
}
long minReferencedGen = outstandingViews.stream().mapToLong(View::minTranslogGeneration).min().orElse(Long.MAX_VALUE);
minReferencedGen = Math.min(lastCommittedTranslogFileGeneration, minReferencedGen);
final long finalMinReferencedGen = minReferencedGen;
List<TranslogReader> unreferenced = readers.stream().filter(r -> r.getGeneration() < finalMinReferencedGen).collect(Collectors.toList());
long minReferencedGen = Math.min(
lastCommittedTranslogFileGeneration,
outstandingViews.stream().mapToLong(View::minTranslogGeneration).min().orElse(Long.MAX_VALUE));
final List<TranslogReader> unreferenced =
readers.stream().filter(r -> r.getGeneration() < minReferencedGen).collect(Collectors.toList());
for (final TranslogReader unreferencedReader : unreferenced) {
Path translogPath = unreferencedReader.path();
logger.trace("delete translog file - not referenced and not current anymore {}", translogPath);
final Path translogPath = unreferencedReader.path();
logger.trace("delete translog file [{}], not referenced and not current anymore", translogPath);
IOUtils.closeWhileHandlingException(unreferencedReader);
IOUtils.deleteFilesIgnoringExceptions(translogPath,
translogPath.resolveSibling(getCommitCheckpointFileName(unreferencedReader.getGeneration())));
@ -1442,13 +1496,6 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
}
}
@Override
public void rollback() throws IOException {
ensureOpen();
close();
}
/**
* References a transaction log generation
*/

20
core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java
View File

@ -37,17 +37,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.UnaryOperator;
import static org.elasticsearch.common.Strings.cleanPath;
public class InternalSettingsPreparer {
private static final String[] ALLOWED_SUFFIXES = {".yml", ".yaml", ".json"};
private static final String PROPERTY_DEFAULTS_PREFIX = "default.";
private static final Predicate<String> PROPERTY_DEFAULTS_PREDICATE = key -> key.startsWith(PROPERTY_DEFAULTS_PREFIX);
private static final UnaryOperator<String> STRIP_PROPERTY_DEFAULTS_PREFIX = key -> key.substring(PROPERTY_DEFAULTS_PREFIX.length());
public static final String SECRET_PROMPT_VALUE = "${prompt.secret}";
public static final String TEXT_PROMPT_VALUE = "${prompt.text}";
@ -125,15 +120,16 @@ public class InternalSettingsPreparer {
}
/**
* Initializes the builder with the given input settings, and loads system properties settings if allowed.
* If loadDefaults is true, system property default settings are loaded.
* Initializes the builder with the given input settings, and applies settings from the specified map (these settings typically come
* from the command line).
*
* @param output the settings builder to apply the input and default settings to
* @param input the input settings
* @param esSettings a map from which to apply settings
*/
private static void initializeSettings(Settings.Builder output, Settings input, Map<String, String> esSettings) {
static void initializeSettings(final Settings.Builder output, final Settings input, final Map<String, String> esSettings) {
output.put(input);
output.putProperties(esSettings,
PROPERTY_DEFAULTS_PREDICATE.and(key -> output.get(STRIP_PROPERTY_DEFAULTS_PREFIX.apply(key)) == null),
STRIP_PROPERTY_DEFAULTS_PREFIX);
output.putProperties(esSettings, PROPERTY_DEFAULTS_PREDICATE.negate(), Function.identity());
output.putProperties(esSettings, Function.identity());
output.replacePropertyPlaceholders();
}

59
core/src/main/java/org/elasticsearch/node/Node.java
View File

@ -50,6 +50,7 @@ import org.elasticsearch.cluster.routing.RoutingService;
import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.StopWatch;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.component.Lifecycle;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.inject.Binder;
@ -58,6 +59,7 @@ import org.elasticsearch.common.inject.Key;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.inject.util.Providers;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.logging.DeprecationLogger;
@ -146,7 +148,9 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
@ -262,6 +266,9 @@ public class Node implements Closeable {
Logger logger = Loggers.getLogger(Node.class, tmpSettings);
final String nodeId = nodeEnvironment.nodeId();
tmpSettings = addNodeNameIfNeeded(tmpSettings, nodeId);
if (DiscoveryNode.nodeRequiresLocalStorage(tmpSettings)) {
checkForIndexDataInDefaultPathData(tmpSettings, nodeEnvironment, logger);
}
// this must be captured after the node name is possibly added to the settings
final String nodeName = NODE_NAME_SETTING.get(tmpSettings);
if (hadPredefinedNodeName == false) {
@ -500,6 +507,58 @@ public class Node implements Closeable {
}
}
/**
* Checks for path.data and default.path.data being configured, and there being index data in any of the paths in default.path.data.
*
* @param settings the settings to check for path.data and default.path.data
* @param nodeEnv the current node environment
* @param logger a logger where messages regarding the detection will be logged
* @throws IOException if an I/O exception occurs reading the directory structure
*/
static void checkForIndexDataInDefaultPathData(
final Settings settings, final NodeEnvironment nodeEnv, final Logger logger) throws IOException {
if (!Environment.PATH_DATA_SETTING.exists(settings) || !Environment.DEFAULT_PATH_DATA_SETTING.exists(settings)) {
return;
}
boolean clean = true;
for (final String defaultPathData : Environment.DEFAULT_PATH_DATA_SETTING.get(settings)) {
final Path nodeDirectory = NodeEnvironment.resolveNodePath(getPath(defaultPathData), nodeEnv.getNodeLockId());
if (Files.exists(nodeDirectory) == false) {
continue;
}
final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(nodeDirectory);
final Set<String> availableIndexFolders = nodeEnv.availableIndexFoldersForPath(nodePath);
if (availableIndexFolders.isEmpty()) {
continue;
}
clean = false;
logger.error("detected index data in default.path.data [{}] where there should not be any", nodePath.indicesPath);
for (final String availableIndexFolder : availableIndexFolders) {
logger.info(
"index folder [{}] in default.path.data [{}] must be moved to any of {}",
availableIndexFolder,
nodePath.indicesPath,
Arrays.stream(nodeEnv.nodePaths()).map(np -> np.indicesPath).collect(Collectors.toList()));
}
}
if (clean) {
return;
}
final String message = String.format(
Locale.ROOT,
"detected index data in default.path.data %s where there should not be any; check the logs for details",
Environment.DEFAULT_PATH_DATA_SETTING.get(settings));
throw new IllegalStateException(message);
}
@SuppressForbidden(reason = "read path that is not configured in environment")
private static Path getPath(final String path) {
return PathUtils.get(path);
}
// visible for testing
static void warnIfPreRelease(final Version version, final boolean isSnapshot, final Logger logger) {
if (!version.isRelease() || isSnapshot) {

19
core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
View File

@ -20,12 +20,14 @@ package org.elasticsearch.rest;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Collections;
import java.util.Set;
import java.util.function.Predicate;
@ -97,7 +99,9 @@ public abstract class AbstractRestChannel implements RestChannel {
excludes = filters.stream().filter(EXCLUDE_FILTER).map(f -> f.substring(1)).collect(toSet());
}
XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(responseContentType), bytesOutput(), includes, excludes);
OutputStream unclosableOutputStream = Streams.flushOnCloseStream(bytesOutput());
XContentBuilder builder =
new XContentBuilder(XContentFactory.xContent(responseContentType), unclosableOutputStream, includes, excludes);
if (pretty) {
builder.prettyPrint().lfAtEnd();
}
@ -107,8 +111,9 @@ public abstract class AbstractRestChannel implements RestChannel {
}
/**
* A channel level bytes output that can be reused. It gets reset on each call to this
* method.
* A channel level bytes output that can be reused. The bytes output is lazily instantiated
* by a call to {@link #newBytesOutput()}. Once the stream is created, it gets reset on each
* call to this method.
*/
@Override
public final BytesStreamOutput bytesOutput() {
@ -120,6 +125,14 @@ public abstract class AbstractRestChannel implements RestChannel {
return bytesOut;
}
/**
* An accessor to the raw value of the channel bytes output. This method will not instantiate
* a new stream if one does not exist and this method will not reset the stream.
*/
protected final BytesStreamOutput bytesOutputOrNull() {
return bytesOut;
}
protected BytesStreamOutput newBytesOutput() {
return new BytesStreamOutput();
}

5
core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
View File

@ -30,7 +30,6 @@ import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
@ -147,8 +146,8 @@ public class BytesRestResponse extends RestResponse {
return builder;
}
static BytesRestResponse createSimpleErrorResponse(RestStatus status, String errorMessage) throws IOException {
return new BytesRestResponse(status, JsonXContent.contentBuilder().startObject()
static BytesRestResponse createSimpleErrorResponse(RestChannel channel, RestStatus status, String errorMessage) throws IOException {
return new BytesRestResponse(status, channel.newErrorBuilder().startObject()
.field("error", errorMessage)
.field("status", status.getStatus())
.endObject());

10
core/src/main/java/org/elasticsearch/rest/RestController.java
View File

@ -178,8 +178,9 @@ public class RestController extends AbstractComponent implements HttpServerTrans
sendContentTypeErrorMessage(request, responseChannel);
} else if (contentLength > 0 && handler != null && handler.supportsContentStream() &&
request.getXContentType() != XContentType.JSON && request.getXContentType() != XContentType.SMILE) {
responseChannel.sendResponse(BytesRestResponse.createSimpleErrorResponse(RestStatus.NOT_ACCEPTABLE, "Content-Type [" +
request.getXContentType() + "] does not support stream parsing. Use JSON or SMILE instead"));
responseChannel.sendResponse(BytesRestResponse.createSimpleErrorResponse(responseChannel,
RestStatus.NOT_ACCEPTABLE, "Content-Type [" + request.getXContentType() +
"] does not support stream parsing. Use JSON or SMILE instead"));
} else {
if (canTripCircuitBreaker(request)) {
inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, "<http_request>");
@ -229,7 +230,8 @@ public class RestController extends AbstractComponent implements HttpServerTrans
void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext,
final RestHandler handler) throws Exception {
if (checkRequestParameters(request, channel) == false) {
channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(BAD_REQUEST, "error traces in responses are disabled."));
channel
.sendResponse(BytesRestResponse.createSimpleErrorResponse(channel,BAD_REQUEST, "error traces in responses are disabled."));
} else {
for (String key : headersToCopy) {
String httpHeader = request.header(key);
@ -283,7 +285,7 @@ public class RestController extends AbstractComponent implements HttpServerTrans
Strings.collectionToCommaDelimitedString(restRequest.getAllHeaderValues("Content-Type")) + "] is not supported";
}
channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(NOT_ACCEPTABLE, errorMessage));
channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(channel, NOT_ACCEPTABLE, errorMessage));
}
/**

5
core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
View File

@ -20,8 +20,9 @@ package org.elasticsearch.rest.action.cat;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.Table;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.UTF8StreamWriter;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.BytesStream;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.BytesRestResponse;
@ -56,7 +57,7 @@ public abstract class AbstractCatAction extends BaseRestHandler {
return channel -> {
Table table = getTableWithHeader(request);
int[] width = buildHelpWidths(table, request);
BytesStreamOutput bytesOutput = channel.bytesOutput();
BytesStream bytesOutput = Streams.flushOnCloseStream(channel.bytesOutput());
UTF8StreamWriter out = new UTF8StreamWriter().setOutput(bytesOutput);
for (Table.Cell cell : table.getHeaders()) {
// need to do left-align always, so create new cells

5
core/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java
View File

@ -22,8 +22,9 @@ package org.elasticsearch.rest.action.cat;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.Table;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.UTF8StreamWriter;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.BytesStream;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.SizeValue;
@ -82,7 +83,7 @@ public class RestTable {
List<DisplayHeader> headers = buildDisplayHeaders(table, request);
int[] width = buildWidths(table, request, verbose, headers);
BytesStreamOutput bytesOut = channel.bytesOutput();
BytesStream bytesOut = Streams.flushOnCloseStream(channel.bytesOutput());
UTF8StreamWriter out = new UTF8StreamWriter().setOutput(bytesOut);
int lastHeader = headers.size() - 1;
if (verbose) {

8
core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java
View File

@ -42,12 +42,12 @@ import java.util.Map;
/**
* A geo metric aggregator that computes a geo-centroid from a {@code geo_point} type field
*/
public final class GeoCentroidAggregator extends MetricsAggregator {
final class GeoCentroidAggregator extends MetricsAggregator {
private final ValuesSource.GeoPoint valuesSource;
LongArray centroids;
LongArray counts;
private LongArray centroids;
private LongArray counts;
protected GeoCentroidAggregator(String name, SearchContext context, Aggregator parent,
GeoCentroidAggregator(String name, SearchContext context, Aggregator parent,
ValuesSource.GeoPoint valuesSource, List<PipelineAggregator> pipelineAggregators,
Map<String, Object> metaData) throws IOException {
super(name, context, parent, pipelineAggregators, metaData);

4
core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorFactory.java
View File

@ -32,9 +32,9 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
public class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, GeoCentroidAggregatorFactory> {
class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, GeoCentroidAggregatorFactory> {
public GeoCentroidAggregatorFactory(String name, ValuesSourceConfig<ValuesSource.GeoPoint> config,
GeoCentroidAggregatorFactory(String name, ValuesSourceConfig<ValuesSource.GeoPoint> config,
SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
Map<String, Object> metaData) throws IOException {
super(name, config, context, parent, subFactoriesBuilder, metaData);

4
core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
View File

@ -38,7 +38,7 @@ public class InternalGeoCentroid extends InternalAggregation implements GeoCentr
protected final GeoPoint centroid;
protected final long count;
public InternalGeoCentroid(String name, GeoPoint centroid, long count, List<PipelineAggregator>
InternalGeoCentroid(String name, GeoPoint centroid, long count, List<PipelineAggregator>
pipelineAggregators, Map<String, Object> metaData) {
super(name, pipelineAggregators, metaData);
assert (centroid == null) == (count == 0);
@ -132,7 +132,7 @@ public class InternalGeoCentroid extends InternalAggregation implements GeoCentr
}
static class Fields {
public static final String CENTROID = "location";
static final String CENTROID = "location";
}
@Override

41
core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/InternalPercentile.java
View File

@ -1,41 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.aggregations.metrics.percentiles;
public class InternalPercentile implements Percentile {
private final double percent;
private final double value;
public InternalPercentile(double percent, double value) {
this.percent = percent;
this.value = value;
}
@Override
public double getPercent() {
return percent;
}
@Override
public double getValue() {
return value;
}
}

37
core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/Percentile.java
View File

@ -19,10 +19,41 @@
package org.elasticsearch.search.aggregations.metrics.percentiles;
public interface Percentile {
import java.util.Objects;
double getPercent();
public class Percentile {
double getValue();
private final double percent;
private final double value;
public Percentile(double percent, double value) {
this.percent = percent;
this.value = value;
}
public double getPercent() {
return percent;
}
public double getValue() {
return value;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Percentile that = (Percentile) o;
return Double.compare(that.percent, percent) == 0
&& Double.compare(that.value, value) == 0;
}
@Override
public int hashCode() {
return Objects.hash(percent, value);
}
}

3
core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
import org.HdrHistogram.DoubleHistogram;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@ -109,7 +108,7 @@ public class InternalHDRPercentileRanks extends AbstractInternalHDRPercentiles i
@Override
public Percentile next() {
final Percentile next = new InternalPercentile(percentileRank(state, values[i]), values[i]);
final Percentile next = new Percentile(percentileRank(state, values[i]), values[i]);
++i;
return next;
}

3
core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
import org.HdrHistogram.DoubleHistogram;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@ -99,7 +98,7 @@ public class InternalHDRPercentiles extends AbstractInternalHDRPercentiles imple
@Override
public Percentile next() {
final Percentile next = new InternalPercentile(percents[i], state.getValueAtPercentile(percents[i]));
final Percentile next = new Percentile(percents[i], state.getValueAtPercentile(percents[i]));
++i;
return next;
}

3
core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java
View File

@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@ -106,7 +105,7 @@ public class InternalTDigestPercentileRanks extends AbstractInternalTDigestPerce
@Override
public Percentile next() {
final Percentile next = new InternalPercentile(percentileRank(state, values[i]), values[i]);
final Percentile next = new Percentile(percentileRank(state, values[i]), values[i]);
++i;
return next;
}

3
core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java
View File

@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@ -95,7 +94,7 @@ public class InternalTDigestPercentiles extends AbstractInternalTDigestPercentil
@Override
public Percentile next() {
final Percentile next = new InternalPercentile(percents[i], state.quantile(percents[i] / 100));
final Percentile next = new Percentile(percents[i], state.quantile(percents[i] / 100));
++i;
return next;
}

3
core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java
View File

@ -26,7 +26,6 @@ import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@ -136,7 +135,7 @@ public class InternalPercentilesBucket extends InternalNumericMetricsAggregation
@Override
public Percentile next() {
final Percentile next = new InternalPercentile(percents[i], percentiles[i]);
final Percentile next = new Percentile(percents[i], percentiles[i]);
++i;
return next;
}

6
core/src/main/java/org/elasticsearch/tasks/TaskInfo.java
View File

@ -183,7 +183,7 @@ public final class TaskInfo implements Writeable, ToXContent {
}
public static final ConstructingObjectParser<TaskInfo, Void> PARSER = new ConstructingObjectParser<>(
"task_info", a -> {
"task_info", true, a -> {
int i = 0;
TaskId id = new TaskId((String) a[i++], (Long) a[i++]);
String type = (String) a[i++];
@ -196,11 +196,11 @@ public final class TaskInfo implements Writeable, ToXContent {
String parentTaskIdString = (String) a[i++];
RawTaskStatus status = statusBytes == null ? null : new RawTaskStatus(statusBytes);
TaskId parentTaskId = parentTaskIdString == null ? TaskId.EMPTY_TASK_ID : new TaskId((String) parentTaskIdString);
TaskId parentTaskId = parentTaskIdString == null ? TaskId.EMPTY_TASK_ID : new TaskId(parentTaskIdString);
return new TaskInfo(id, type, action, description, status, startTime, runningTimeNanos, cancellable, parentTaskId);
});
static {
// Note for the future: this has to be backwards compatible with all changes to the task storage format
// Note for the future: this has to be backwards and forwards compatible with all changes to the task storage format
PARSER.declareString(constructorArg(), new ParseField("node"));
PARSER.declareLong(constructorArg(), new ParseField("id"));
PARSER.declareString(constructorArg(), new ParseField("type"));

33
core/src/main/java/org/elasticsearch/transport/TcpTransport.java
View File

@ -41,7 +41,7 @@ import org.elasticsearch.common.component.Lifecycle;
import org.elasticsearch.common.compress.Compressor;
import org.elasticsearch.common.compress.CompressorFactory;
import org.elasticsearch.common.compress.NotCompressedException;
import org.elasticsearch.common.io.ReleasableBytesStream;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@ -1025,10 +1025,8 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
}
status = TransportStatus.setRequest(status);
ReleasableBytesStreamOutput bStream = new ReleasableBytesStreamOutput(bigArrays);
// we wrap this in a release once since if the onRequestSent callback throws an exception
// we might release things twice and this should be prevented
final Releasable toRelease = Releasables.releaseOnce(() -> Releasables.close(bStream.bytes()));
StreamOutput stream = bStream;
boolean addedReleaseListener = false;
StreamOutput stream = Streams.flushOnCloseStream(bStream);
try {
// only compress if asked, and, the request is not bytes, since then only
// the header part is compressed, and the "body" can't be extracted as compressed
@ -1047,12 +1045,17 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
stream.writeString(action);
BytesReference message = buildMessage(requestId, status, node.getVersion(), request, stream, bStream);
final TransportRequestOptions finalOptions = options;
final StreamOutput finalStream = stream;
// this might be called in a different thread
SendListener onRequestSent = new SendListener(toRelease,
SendListener onRequestSent = new SendListener(
() -> IOUtils.closeWhileHandlingException(finalStream, bStream),
() -> transportServiceAdapter.onRequestSent(node, requestId, action, request, finalOptions));
internalSendMessage(targetChannel, message, onRequestSent);
addedReleaseListener = true;
} finally {
IOUtils.close(stream);
if (!addedReleaseListener) {
IOUtils.close(stream, bStream);
}
}
}
@ -1114,10 +1117,8 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
}
status = TransportStatus.setResponse(status); // TODO share some code with sendRequest
ReleasableBytesStreamOutput bStream = new ReleasableBytesStreamOutput(bigArrays);
// we wrap this in a release once since if the onRequestSent callback throws an exception
// we might release things twice and this should be prevented
final Releasable toRelease = Releasables.releaseOnce(() -> Releasables.close(bStream.bytes()));
StreamOutput stream = bStream;
boolean addedReleaseListener = false;
StreamOutput stream = Streams.flushOnCloseStream(bStream);
try {
if (options.compress()) {
status = TransportStatus.setCompress(status);
@ -1128,12 +1129,16 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
BytesReference reference = buildMessage(requestId, status, nodeVersion, response, stream, bStream);
final TransportResponseOptions finalOptions = options;
final StreamOutput finalStream = stream;
// this might be called in a different thread
SendListener listener = new SendListener(toRelease,
SendListener listener = new SendListener(() -> IOUtils.closeWhileHandlingException(finalStream, bStream),
() -> transportServiceAdapter.onResponseSent(requestId, action, response, finalOptions));
internalSendMessage(channel, reference, listener);
addedReleaseListener = true;
} finally {
IOUtils.close(stream);
if (!addedReleaseListener) {
IOUtils.close(stream, bStream);
}
}
}
@ -1161,7 +1166,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
* Serializes the given message into a bytes representation
*/
private BytesReference buildMessage(long requestId, byte status, Version nodeVersion, TransportMessage message, StreamOutput stream,
ReleasableBytesStream writtenBytes) throws IOException {
ReleasableBytesStreamOutput writtenBytes) throws IOException {
final BytesReference zeroCopyBuffer;
if (message instanceof BytesTransportRequest) { // what a shitty optimization - we should use a direct send method instead
BytesTransportRequest bRequest = (BytesTransportRequest) message;

36
core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java
View File

@ -80,43 +80,37 @@ public class RetryTests extends ESTestCase {
return request;
}
public void testSyncRetryBacksOff() throws Exception {
public void testRetryBacksOff() throws Exception {
BackoffPolicy backoff = BackoffPolicy.constantBackoff(DELAY, CALLS_TO_FAIL);
BulkRequest bulkRequest = createBulkRequest();
BulkResponse response = Retry
.on(EsRejectedExecutionException.class)
.policy(backoff)
.using(bulkClient.threadPool())
.withSyncBackoff(bulkClient::bulk, bulkRequest, bulkClient.settings());
BulkResponse response = new Retry(EsRejectedExecutionException.class, backoff, bulkClient.threadPool())
.withBackoff(bulkClient::bulk, bulkRequest, bulkClient.settings())
.actionGet();
assertFalse(response.hasFailures());
assertThat(response.getItems().length, equalTo(bulkRequest.numberOfActions()));
}
public void testSyncRetryFailsAfterBackoff() throws Exception {
public void testRetryFailsAfterBackoff() throws Exception {
BackoffPolicy backoff = BackoffPolicy.constantBackoff(DELAY, CALLS_TO_FAIL - 1);
BulkRequest bulkRequest = createBulkRequest();
BulkResponse response = Retry
.on(EsRejectedExecutionException.class)
.policy(backoff)
.using(bulkClient.threadPool())
.withSyncBackoff(bulkClient::bulk, bulkRequest, bulkClient.settings());
BulkResponse response = new Retry(EsRejectedExecutionException.class, backoff, bulkClient.threadPool())
.withBackoff(bulkClient::bulk, bulkRequest, bulkClient.settings())
.actionGet();
assertTrue(response.hasFailures());
assertThat(response.getItems().length, equalTo(bulkRequest.numberOfActions()));
}
public void testAsyncRetryBacksOff() throws Exception {
public void testRetryWithListenerBacksOff() throws Exception {
BackoffPolicy backoff = BackoffPolicy.constantBackoff(DELAY, CALLS_TO_FAIL);
AssertingListener listener = new AssertingListener();
BulkRequest bulkRequest = createBulkRequest();
Retry.on(EsRejectedExecutionException.class)
.policy(backoff)
.using(bulkClient.threadPool())
.withAsyncBackoff(bulkClient::bulk, bulkRequest, listener, bulkClient.settings());
Retry retry = new Retry(EsRejectedExecutionException.class, backoff, bulkClient.threadPool());
retry.withBackoff(bulkClient::bulk, bulkRequest, listener, bulkClient.settings());
listener.awaitCallbacksCalled();
listener.assertOnResponseCalled();
@ -125,15 +119,13 @@ public class RetryTests extends ESTestCase {
listener.assertOnFailureNeverCalled();
}
public void testAsyncRetryFailsAfterBacksOff() throws Exception {
public void testRetryWithListenerFailsAfterBacksOff() throws Exception {
BackoffPolicy backoff = BackoffPolicy.constantBackoff(DELAY, CALLS_TO_FAIL - 1);
AssertingListener listener = new AssertingListener();
BulkRequest bulkRequest = createBulkRequest();
Retry.on(EsRejectedExecutionException.class)
.policy(backoff)
.using(bulkClient.threadPool())
.withAsyncBackoff(bulkClient::bulk, bulkRequest, listener, bulkClient.settings());
Retry retry = new Retry(EsRejectedExecutionException.class, backoff, bulkClient.threadPool());
retry.withBackoff(bulkClient::bulk, bulkRequest, listener, bulkClient.settings());
listener.awaitCallbacksCalled();

1
core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
View File

@ -166,6 +166,7 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
Files.createDirectories(multiDataPath[0]);
Files.createDirectories(multiDataPath[1]);
logger.info("--> Multi data paths: {}, {}", multiDataPath[0], multiDataPath[1]);
ensureGreen();
}
void upgradeIndexFolder() throws Exception {

51
core/src/test/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutputTests.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.io.stream;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
public class ReleasableBytesStreamOutputTests extends ESTestCase {
public void testRelease() throws Exception {
MockBigArrays mockBigArrays =
new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
try (ReleasableBytesStreamOutput output =
getRandomReleasableBytesStreamOutput(mockBigArrays)) {
output.writeBoolean(randomBoolean());
}
MockBigArrays.ensureAllArraysAreReleased();
}
private ReleasableBytesStreamOutput getRandomReleasableBytesStreamOutput(
MockBigArrays mockBigArrays) throws IOException {
ReleasableBytesStreamOutput output = new ReleasableBytesStreamOutput(mockBigArrays);
if (randomBoolean()) {
for (int i = 0; i < scaledRandomIntBetween(1, 32); i++) {
output.write(randomByte());
}
}
return output;
}
}

6
core/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
View File

@ -31,7 +31,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.uid.VersionsResolver.DocIdAndVersion;
import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion;
import org.elasticsearch.index.mapper.UidFieldMapper;
import org.elasticsearch.index.mapper.VersionFieldMapper;
import org.elasticsearch.test.ESTestCase;
@ -53,7 +53,7 @@ public class VersionLookupTests extends ESTestCase {
writer.addDocument(doc);
DirectoryReader reader = DirectoryReader.open(writer);
LeafReaderContext segment = reader.leaves().get(0);
PerThreadIDAndVersionLookup lookup = new PerThreadIDAndVersionLookup(segment.reader());
PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader());
// found doc
DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), null, segment);
assertNotNull(result);
@ -81,7 +81,7 @@ public class VersionLookupTests extends ESTestCase {
writer.addDocument(doc);
DirectoryReader reader = DirectoryReader.open(writer);
LeafReaderContext segment = reader.leaves().get(0);
PerThreadIDAndVersionLookup lookup = new PerThreadIDAndVersionLookup(segment.reader());
PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader());
// return the last doc when there are duplicates
DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), null, segment);
assertNotNull(result);

20
core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
View File

@ -38,8 +38,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.elasticsearch.common.lucene.uid.VersionsResolver.loadDocIdAndVersion;
import static org.elasticsearch.common.lucene.uid.VersionsResolver.loadVersion;
import static org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.loadDocIdAndVersion;
import static org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.loadVersion;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
@ -145,7 +145,7 @@ public class VersionsTests extends ESTestCase {
/** Test that version map cache works, is evicted on close, etc */
public void testCache() throws Exception {
int size = VersionsResolver.lookupStates.size();
int size = VersionsAndSeqNoResolver.lookupStates.size();
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
@ -156,21 +156,21 @@ public class VersionsTests extends ESTestCase {
DirectoryReader reader = DirectoryReader.open(writer);
// should increase cache size by 1
assertEquals(87, loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
assertEquals(size+1, VersionsResolver.lookupStates.size());
assertEquals(size+1, VersionsAndSeqNoResolver.lookupStates.size());
// should be cache hit
assertEquals(87, loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
assertEquals(size+1, VersionsResolver.lookupStates.size());
assertEquals(size+1, VersionsAndSeqNoResolver.lookupStates.size());
reader.close();
writer.close();
// core should be evicted from the map
assertEquals(size, VersionsResolver.lookupStates.size());
assertEquals(size, VersionsAndSeqNoResolver.lookupStates.size());
dir.close();
}
/** Test that version map cache behaves properly with a filtered reader */
public void testCacheFilterReader() throws Exception {
int size = VersionsResolver.lookupStates.size();
int size = VersionsAndSeqNoResolver.lookupStates.size();
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
@ -180,17 +180,17 @@ public class VersionsTests extends ESTestCase {
writer.addDocument(doc);
DirectoryReader reader = DirectoryReader.open(writer);
assertEquals(87, loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
assertEquals(size+1, VersionsResolver.lookupStates.size());
assertEquals(size+1, VersionsAndSeqNoResolver.lookupStates.size());
// now wrap the reader
DirectoryReader wrapped = ElasticsearchDirectoryReader.wrap(reader, new ShardId("bogus", "_na_", 5));
assertEquals(87, loadVersion(wrapped, new Term(UidFieldMapper.NAME, "6")));
// same size map: core cache key is shared
assertEquals(size+1, VersionsResolver.lookupStates.size());
assertEquals(size+1, VersionsAndSeqNoResolver.lookupStates.size());
reader.close();
writer.close();
// core should be evicted from the map
assertEquals(size, VersionsResolver.lookupStates.size());
assertEquals(size, VersionsAndSeqNoResolver.lookupStates.size());
dir.close();
}
}

12
core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
View File

@ -562,4 +562,16 @@ public class SettingsTests extends ESTestCase {
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> setting.get(settings));
assertTrue(e.getMessage().contains("must be stored inside the Elasticsearch keystore"));
}
public void testGetAsArrayFailsOnDuplicates() {
final Settings settings =
Settings.builder()
.put("foobar.0", "bar")
.put("foobar.1", "baz")
.put("foobar", "foo")
.build();
final IllegalStateException e = expectThrows(IllegalStateException.class, () -> settings.getAsArray("foobar"));
assertThat(e, hasToString(containsString("settings object contains values for [foobar=foo] and [foobar.0=bar]")));
}
}

15
core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
View File

@ -139,4 +139,19 @@ public class ElectMasterServiceTests extends ESTestCase {
}
}
}
public void testCountMasterNodes() {
List<DiscoveryNode> nodes = generateRandomNodes();
ElectMasterService service = electMasterService();
int masterNodes = 0;
for (DiscoveryNode node : nodes) {
if (node.isMasterNode()) {
masterNodes++;
}
}
assertEquals(masterNodes, service.countMasterNodes(nodes));
}
}

2
core/src/test/java/org/elasticsearch/discovery/zen/NodeRemovalClusterStateTaskExecutorTests.java
View File

@ -108,6 +108,8 @@ public class NodeRemovalClusterStateTaskExecutorTests extends ESTestCase {
final ClusterStateTaskExecutor.ClusterTasksResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result =
executor.execute(clusterState, tasks);
verify(electMasterService).hasEnoughMasterNodes(eq(remainingNodesClusterState.get().nodes()));
verify(electMasterService).countMasterNodes(eq(remainingNodesClusterState.get().nodes()));
verify(electMasterService).minimumMasterNodes();
verifyNoMoreInteractions(electMasterService);
// ensure that we did not reroute

102
core/src/test/java/org/elasticsearch/env/EnvironmentTests.java vendored
View File

@ -23,10 +23,12 @@ import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Path;
import static org.hamcrest.CoreMatchers.endsWith;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.Matchers.equalTo;
/**
* Simple unit-tests for Environment.java
@ -71,4 +73,104 @@ public class EnvironmentTests extends ESTestCase {
assertThat(environment.resolveRepoURL(new URL("jar:http://localhost/test/../repo1?blah!/repo/")), nullValue());
}
public void testDefaultPathData() {
final Path defaultPathData = createTempDir().toAbsolutePath();
final Settings settings = Settings.builder()
.put("path.home", createTempDir().toAbsolutePath())
.put("default.path.data", defaultPathData)
.build();
final Environment environment = new Environment(settings);
assertThat(environment.dataFiles(), equalTo(new Path[] { defaultPathData }));
}
public void testPathDataOverrideDefaultPathData() {
final Path pathData = createTempDir().toAbsolutePath();
final Path defaultPathData = createTempDir().toAbsolutePath();
final Settings settings = Settings.builder()
.put("path.home", createTempDir().toAbsolutePath())
.put("path.data", pathData)
.put("default.path.data", defaultPathData)
.build();
final Environment environment = new Environment(settings);
assertThat(environment.dataFiles(), equalTo(new Path[] { pathData }));
}
public void testPathDataWhenNotSet() {
final Path pathHome = createTempDir().toAbsolutePath();
final Settings settings = Settings.builder().put("path.home", pathHome).build();
final Environment environment = new Environment(settings);
assertThat(environment.dataFiles(), equalTo(new Path[]{pathHome.resolve("data")}));
}
public void testPathDataNotSetInEnvironmentIfNotSet() {
final Path defaultPathData = createTempDir().toAbsolutePath();
final Settings settings = Settings.builder()
.put("path.home", createTempDir().toAbsolutePath())
.put("default.path.data", defaultPathData)
.build();
assertFalse(Environment.PATH_DATA_SETTING.exists(settings));
assertTrue(Environment.DEFAULT_PATH_DATA_SETTING.exists(settings));
final Environment environment = new Environment(settings);
assertFalse(Environment.PATH_DATA_SETTING.exists(environment.settings()));
assertTrue(Environment.DEFAULT_PATH_DATA_SETTING.exists(environment.settings()));
}
public void testDefaultPathLogs() {
final Path defaultPathLogs = createTempDir().toAbsolutePath();
final Settings settings = Settings.builder()
.put("path.home", createTempDir().toAbsolutePath())
.put("default.path.logs", defaultPathLogs)
.build();
final Environment environment = new Environment(settings);
assertThat(environment.logsFile(), equalTo(defaultPathLogs));
}
public void testPathLogsOverrideDefaultPathLogs() {
final Path pathLogs = createTempDir().toAbsolutePath();
final Path defaultPathLogs = createTempDir().toAbsolutePath();
final Settings settings = Settings.builder()
.put("path.home", createTempDir().toAbsolutePath())
.put("path.logs", pathLogs)
.put("default.path.logs", defaultPathLogs)
.build();
final Environment environment = new Environment(settings);
assertThat(environment.logsFile(), equalTo(pathLogs));
}
public void testPathLogsWhenNotSet() {
final Path pathHome = createTempDir().toAbsolutePath();
final Settings settings = Settings.builder().put("path.home", pathHome).build();
final Environment environment = new Environment(settings);
assertThat(environment.logsFile(), equalTo(pathHome.resolve("logs")));
}
public void testDefaultPathConf() {
final Path defaultPathConf = createTempDir().toAbsolutePath();
final Settings settings = Settings.builder()
.put("path.home", createTempDir().toAbsolutePath())
.put("default.path.conf", defaultPathConf)
.build();
final Environment environment = new Environment(settings);
assertThat(environment.configFile(), equalTo(defaultPathConf));
}
public void testPathConfOverrideDefaultPathConf() {
final Path pathConf = createTempDir().toAbsolutePath();
final Path defaultPathConf = createTempDir().toAbsolutePath();
final Settings settings = Settings.builder()
.put("path.home", createTempDir().toAbsolutePath())
.put("path.conf", pathConf)
.put("default.path.conf", defaultPathConf)
.build();
final Environment environment = new Environment(settings);
assertThat(environment.configFile(), equalTo(pathConf));
}
public void testPathConfWhenNotSet() {
final Path pathHome = createTempDir().toAbsolutePath();
final Settings settings = Settings.builder().put("path.home", pathHome).build();
final Environment environment = new Environment(settings);
assertThat(environment.configFile(), equalTo(pathHome.resolve("config")));
}
}

2
core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
View File

@ -41,7 +41,7 @@ import static org.hamcrest.Matchers.startsWith;
public class IndexingSlowLogTests extends ESTestCase {
public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException {
BytesReference source = JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes();
ParsedDocument pd = new ParsedDocument(new NumericDocValuesField("version", 1), SeqNoFieldMapper.SequenceID.emptySeqID(), "id",
ParsedDocument pd = new ParsedDocument(new NumericDocValuesField("version", 1), SeqNoFieldMapper.SequenceIDFields.emptySeqID(), "id",
"test", null, null, source, XContentType.JSON, null);
Index index = new Index("foo", "123");
// Turning off document logging doesn't log source[]

237
core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
View File

@ -83,7 +83,8 @@ import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.uid.Versions;
import org.elasticsearch.common.lucene.uid.VersionsResolver;
import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver;
import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.BigArrays;
@ -150,6 +151,7 @@ import java.util.Base64;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@ -165,6 +167,8 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
import static java.util.Collections.emptyMap;
import static java.util.Collections.shuffle;
@ -292,7 +296,7 @@ public class InternalEngineTests extends ESTestCase {
private static ParsedDocument testParsedDocument(String id, String type, String routing, Document document, BytesReference source, Mapping mappingUpdate) {
Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
Field versionField = new NumericDocValuesField("_version", 0);
SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
document.add(uidField);
document.add(versionField);
document.add(seqID.seqNo);
@ -833,6 +837,58 @@ public class InternalEngineTests extends ESTestCase {
}
}
public void testTranslogRecoveryWithMultipleGenerations() throws IOException {
final int docs = randomIntBetween(1, 4096);
final List<Long> seqNos = LongStream.range(0, docs).boxed().collect(Collectors.toList());
Randomness.shuffle(seqNos);
engine.close();
Engine initialEngine = null;
try {
final AtomicInteger counter = new AtomicInteger();
initialEngine = new InternalEngine(copy(engine.config(), EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG)) {
@Override
public SequenceNumbersService seqNoService() {
return new SequenceNumbersService(
engine.shardId,
engine.config().getIndexSettings(),
SequenceNumbersService.NO_OPS_PERFORMED,
SequenceNumbersService.NO_OPS_PERFORMED,
SequenceNumbersService.UNASSIGNED_SEQ_NO) {
@Override
public long generateSeqNo() {
return seqNos.get(counter.getAndIncrement());
}
};
}
};
for (int i = 0; i < docs; i++) {
final String id = Integer.toString(i);
final ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null);
initialEngine.index(indexForDoc(doc));
if (rarely()) {
initialEngine.getTranslog().rollGeneration();
} else if (rarely()) {
initialEngine.flush();
}
}
} finally {
IOUtils.close(initialEngine);
}
Engine recoveringEngine = null;
try {
recoveringEngine = new InternalEngine(copy(initialEngine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG));
recoveringEngine.recoverFromTranslog();
try (Engine.Searcher searcher = recoveringEngine.acquireSearcher("test")) {
TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), docs);
assertEquals(docs, topDocs.totalHits);
}
} finally {
IOUtils.close(recoveringEngine);
}
}
public void testConcurrentGetAndFlush() throws Exception {
ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
engine.index(indexForDoc(doc));
@ -1369,19 +1425,10 @@ public class InternalEngineTests extends ESTestCase {
public void testOutOfOrderDocsOnReplica() throws IOException {
final List<Engine.Operation> ops = generateSingleDocHistory(true,
randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), false, 2, 2, 20);
randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL, VersionType.EXTERNAL_GTE, VersionType.FORCE), false, 2, 2, 20);
assertOpsOnReplica(ops, replicaEngine, true);
}
public void testNonStandardVersioningOnReplica() throws IOException {
// TODO: this can be folded into testOutOfOrderDocsOnReplica once out of order
// is detected using seq#
final List<Engine.Operation> ops = generateSingleDocHistory(true,
randomFrom(VersionType.EXTERNAL_GTE, VersionType.FORCE), false, 2, 2, 20);
assertOpsOnReplica(ops, replicaEngine, false);
}
public void testOutOfOrderDocsOnReplicaOldPrimary() throws IOException {
IndexSettings oldSettings = IndexSettingsModule.newIndexSettings("testOld", Settings.builder()
.put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), "1h") // make sure this doesn't kick in on us
@ -3357,48 +3404,68 @@ public class InternalEngineTests extends ESTestCase {
searchResult.close();
}
/**
* A sequence number service that will generate a sequence number and if {@code stall} is set to {@code true} will wait on the barrier
* and the referenced latch before returning. If the local checkpoint should advance (because {@code stall} is {@code false}), then the
* value of {@code expectedLocalCheckpoint} is set accordingly.
*
* @param latchReference to latch the thread for the purpose of stalling
* @param barrier to signal the thread has generated a new sequence number
* @param stall whether or not the thread should stall
* @param expectedLocalCheckpoint the expected local checkpoint after generating a new sequence
* number
* @return a sequence number service
*/
private SequenceNumbersService getStallingSeqNoService(
final AtomicReference<CountDownLatch> latchReference,
final CyclicBarrier barrier,
final AtomicBoolean stall,
final AtomicLong expectedLocalCheckpoint) {
return new SequenceNumbersService(
shardId,
defaultSettings,
SequenceNumbersService.NO_OPS_PERFORMED,
SequenceNumbersService.NO_OPS_PERFORMED,
SequenceNumbersService.UNASSIGNED_SEQ_NO) {
@Override
public long generateSeqNo() {
final long seqNo = super.generateSeqNo();
final CountDownLatch latch = latchReference.get();
if (stall.get()) {
try {
barrier.await();
latch.await();
} catch (BrokenBarrierException | InterruptedException e) {
throw new RuntimeException(e);
}
} else {
if (expectedLocalCheckpoint.get() + 1 == seqNo) {
expectedLocalCheckpoint.set(seqNo);
}
}
return seqNo;
}
};
}
public void testSequenceNumberAdvancesToMaxSeqOnEngineOpenOnPrimary() throws BrokenBarrierException, InterruptedException, IOException {
engine.close();
final int docs = randomIntBetween(1, 32);
InternalEngine initialEngine = null;
try {
final CountDownLatch latch = new CountDownLatch(1);
final AtomicReference<CountDownLatch> latchReference = new AtomicReference<>(new CountDownLatch(1));
final CyclicBarrier barrier = new CyclicBarrier(2);
final AtomicBoolean skip = new AtomicBoolean();
final AtomicBoolean stall = new AtomicBoolean();
final AtomicLong expectedLocalCheckpoint = new AtomicLong(SequenceNumbersService.NO_OPS_PERFORMED);
final List<Thread> threads = new ArrayList<>();
final SequenceNumbersService seqNoService =
new SequenceNumbersService(
shardId,
defaultSettings,
SequenceNumbersService.NO_OPS_PERFORMED,
SequenceNumbersService.NO_OPS_PERFORMED,
SequenceNumbersService.UNASSIGNED_SEQ_NO) {
@Override
public long generateSeqNo() {
final long seqNo = super.generateSeqNo();
if (skip.get()) {
try {
barrier.await();
latch.await();
} catch (BrokenBarrierException | InterruptedException e) {
throw new RuntimeException(e);
}
} else {
if (expectedLocalCheckpoint.get() + 1 == seqNo) {
expectedLocalCheckpoint.set(seqNo);
}
}
return seqNo;
}
};
final SequenceNumbersService seqNoService = getStallingSeqNoService(latchReference, barrier, stall, expectedLocalCheckpoint);
initialEngine = createEngine(defaultSettings, store, primaryTranslogDir, newMergePolicy(), null, () -> seqNoService);
final InternalEngine finalInitialEngine = initialEngine;
for (int i = 0; i < docs; i++) {
final String id = Integer.toString(i);
final ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null);
skip.set(randomBoolean());
stall.set(randomBoolean());
final Thread thread = new Thread(() -> {
try {
finalInitialEngine.index(indexForDoc(doc));
@ -3407,7 +3474,7 @@ public class InternalEngineTests extends ESTestCase {
}
});
thread.start();
if (skip.get()) {
if (stall.get()) {
threads.add(thread);
barrier.await();
} else {
@ -3419,7 +3486,7 @@ public class InternalEngineTests extends ESTestCase {
assertThat(initialEngine.seqNoService().getMaxSeqNo(), equalTo((long) (docs - 1)));
initialEngine.flush(true, true);
latch.countDown();
latchReference.get().countDown();
for (final Thread thread : threads) {
thread.join();
}
@ -3594,6 +3661,78 @@ public class InternalEngineTests extends ESTestCase {
}
}
public void testMinGenerationForSeqNo() throws IOException, BrokenBarrierException, InterruptedException {
engine.close();
final int numberOfTriplets = randomIntBetween(1, 32);
InternalEngine actualEngine = null;
try {
final AtomicReference<CountDownLatch> latchReference = new AtomicReference<>();
final CyclicBarrier barrier = new CyclicBarrier(2);
final AtomicBoolean stall = new AtomicBoolean();
final AtomicLong expectedLocalCheckpoint = new AtomicLong(SequenceNumbersService.NO_OPS_PERFORMED);
final Map<Thread, CountDownLatch> threads = new LinkedHashMap<>();
final SequenceNumbersService seqNoService = getStallingSeqNoService(latchReference, barrier, stall, expectedLocalCheckpoint);
actualEngine = createEngine(defaultSettings, store, primaryTranslogDir, newMergePolicy(), null, () -> seqNoService);
final InternalEngine finalActualEngine = actualEngine;
final Translog translog = finalActualEngine.getTranslog();
final long generation = finalActualEngine.getTranslog().currentFileGeneration();
for (int i = 0; i < numberOfTriplets; i++) {
/*
* Index three documents with the first and last landing in the same generation and the middle document being stalled until
* a later generation.
*/
stall.set(false);
index(finalActualEngine, 3 * i);
final CountDownLatch latch = new CountDownLatch(1);
latchReference.set(latch);
final int skipId = 3 * i + 1;
stall.set(true);
final Thread thread = new Thread(() -> {
try {
index(finalActualEngine, skipId);
} catch (IOException e) {
throw new AssertionError(e);
}
});
thread.start();
threads.put(thread, latch);
barrier.await();
stall.set(false);
index(finalActualEngine, 3 * i + 2);
finalActualEngine.flush();
/*
* This sequence number landed in the last generation, but the lower and upper bounds for an earlier generation straddle
* this sequence number.
*/
assertThat(translog.getMinGenerationForSeqNo(3 * i + 1).translogFileGeneration, equalTo(i + generation));
}
int i = 0;
for (final Map.Entry<Thread, CountDownLatch> entry : threads.entrySet()) {
final Map<String, String> userData = finalActualEngine.commitStats().getUserData();
assertThat(userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY), equalTo(Long.toString(3 * i)));
assertThat(userData.get(Translog.TRANSLOG_GENERATION_KEY), equalTo(Long.toString(i + generation)));
entry.getValue().countDown();
entry.getKey().join();
finalActualEngine.flush();
i++;
}
} finally {
IOUtils.close(actualEngine);
}
}
private void index(final InternalEngine engine, final int id) throws IOException {
final String docId = Integer.toString(id);
final ParsedDocument doc =
testParsedDocument(docId, "test", null, testDocumentWithTextField(), SOURCE, null);
engine.index(indexForDoc(doc));
}
/**
* Return a tuple representing the sequence ID for the given {@code Get}
* operation. The first value in the tuple is the sequence number, the
@ -3601,9 +3740,17 @@ public class InternalEngineTests extends ESTestCase {
*/
private Tuple<Long, Long> getSequenceID(Engine engine, Engine.Get get) throws EngineException {
try (Searcher searcher = engine.acquireSearcher("get")) {
long seqNum = VersionsResolver.loadSeqNo(searcher.reader(), get.uid());
long primaryTerm = VersionsResolver.loadPrimaryTerm(searcher.reader(), get.uid());
return new Tuple<>(seqNum, primaryTerm);
final long primaryTerm;
final long seqNo;
DocIdAndSeqNo docIdAndSeqNo = VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.reader(), get.uid());
if (docIdAndSeqNo == null) {
primaryTerm = 0;
seqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO;
} else {
seqNo = docIdAndSeqNo.seqNo;
primaryTerm = VersionsAndSeqNoResolver.loadPrimaryTerm(docIdAndSeqNo);
}
return new Tuple<>(seqNo, primaryTerm);
} catch (Exception e) {
throw new EngineException(shardId, "unable to retrieve sequence id", e);
}

4
core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
View File

@ -33,7 +33,7 @@ public class LiveVersionMapTests extends ESTestCase {
for (int i = 0; i < 100000; ++i) {
BytesRefBuilder uid = new BytesRefBuilder();
uid.copyChars(TestUtil.randomSimpleString(random(), 10, 20));
VersionValue version = new VersionValue(randomLong());
VersionValue version = new VersionValue(randomLong(), randomLong(), randomLong());
map.putUnderLock(uid.toBytesRef(), version);
}
long actualRamBytesUsed = RamUsageTester.sizeOf(map);
@ -48,7 +48,7 @@ public class LiveVersionMapTests extends ESTestCase {
for (int i = 0; i < 100000; ++i) {
BytesRefBuilder uid = new BytesRefBuilder();
uid.copyChars(TestUtil.randomSimpleString(random(), 10, 20));
VersionValue version = new VersionValue(randomLong());
VersionValue version = new VersionValue(randomLong(), randomLong(), randomLong());
map.putUnderLock(uid.toBytesRef(), version);
}
actualRamBytesUsed = RamUsageTester.sizeOf(map);

4
core/src/test/java/org/elasticsearch/index/engine/VersionValueTests.java
View File

@ -25,12 +25,12 @@ import org.elasticsearch.test.ESTestCase;
public class VersionValueTests extends ESTestCase {
public void testRamBytesUsed() {
VersionValue versionValue = new VersionValue(randomLong());
VersionValue versionValue = new VersionValue(randomLong(), randomLong(), randomLong());
assertEquals(RamUsageTester.sizeOf(versionValue), versionValue.ramBytesUsed());
}
public void testDeleteRamBytesUsed() {
DeleteVersionValue versionValue = new DeleteVersionValue(randomLong(), randomLong());
DeleteVersionValue versionValue = new DeleteVersionValue(randomLong(), randomLong(), randomLong(), randomLong());
assertEquals(RamUsageTester.sizeOf(versionValue), versionValue.ramBytesUsed());
}

26
core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
View File

@ -27,13 +27,9 @@ import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.bulk.BulkItemRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.bulk.BulkShardRequest;
import org.elasticsearch.action.bulk.BulkShardResponse;
import org.elasticsearch.action.bulk.TransportShardBulkActionTests;
import org.elasticsearch.action.bulk.TransportSingleItemBulkWriteAction;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.PlainActionFuture;
@ -98,6 +94,10 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
}
protected IndexMetaData buildIndexMetaData(int replicas) throws IOException {
return buildIndexMetaData(replicas, indexMapping);
}
protected IndexMetaData buildIndexMetaData(int replicas, Map<String, String> mappings) throws IOException {
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
@ -105,7 +105,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
IndexMetaData.Builder metaData = IndexMetaData.builder(index.getName())
.settings(settings)
.primaryTerm(0, 1);
for (Map.Entry<String, String> typeMapping : indexMapping.entrySet()) {
for (Map.Entry<String, String> typeMapping : mappings.entrySet()) {
metaData.putMapping(typeMapping.getKey(), typeMapping.getValue());
}
return metaData.build();
@ -224,15 +224,24 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
updateAllocationIDsOnPrimary();
}
public synchronized IndexShard addReplica() throws IOException {
public IndexShard addReplica() throws IOException {
final ShardRouting replicaRouting = createShardRouting("s" + replicaId.incrementAndGet(), false);
final IndexShard replica =
newShard(replicaRouting, indexMetaData, null, this::syncGlobalCheckpoint, getEngineFactory(replicaRouting));
replicas.add(replica);
updateAllocationIDsOnPrimary();
addReplica(replica);
return replica;
}
public synchronized void addReplica(IndexShard replica) {
assert shardRoutings().stream()
.filter(shardRouting -> shardRouting.isSameAllocation(replica.routingEntry())).findFirst().isPresent() == false :
"replica with aId [" + replica.routingEntry().allocationId() + "] already exists";
replica.updatePrimaryTerm(primary.getPrimaryTerm());
replicas.add(replica);
updateAllocationIDsOnPrimary();
}
public synchronized IndexShard addReplicaWithExistingPath(final ShardPath shardPath, final String nodeId) throws IOException {
final ShardRouting shardRouting = TestShardRouting.newShardRouting(
shardId,
@ -264,6 +273,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
}
boolean found = replicas.remove(replica);
assert found;
closeShards(primary);
primary = replica;
replica.updateRoutingEntry(replica.routingEntry().moveActiveReplicaToPrimary());
updateAllocationIDsOnPrimary();

28
core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
View File

@ -18,6 +18,9 @@
*/
package org.elasticsearch.index.replication;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
@ -37,6 +40,7 @@ import org.elasticsearch.indices.recovery.RecoveryTarget;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
@ -152,4 +156,28 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase
}
}
public void testConflictingOpsOnReplica() throws Exception {
Map<String, String> mappings =
Collections.singletonMap("type", "{ \"type\": { \"properties\": { \"f\": { \"type\": \"keyword\"} }}}");
try (ReplicationGroup shards = new ReplicationGroup(buildIndexMetaData(2, mappings))) {
shards.startAll();
IndexShard replica1 = shards.getReplicas().get(0);
logger.info("--> isolated replica " + replica1.routingEntry());
shards.removeReplica(replica1);
IndexRequest indexRequest = new IndexRequest(index.getName(), "type", "1").source("{ \"f\": \"1\"}", XContentType.JSON);
shards.index(indexRequest);
shards.addReplica(replica1);
logger.info("--> promoting replica to primary " + replica1.routingEntry());
shards.promoteReplicaToPrimary(replica1);
indexRequest = new IndexRequest(index.getName(), "type", "1").source("{ \"f\": \"2\"}", XContentType.JSON);
shards.index(indexRequest);
shards.refresh("test");
for (IndexShard shard : shards) {
try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
TopDocs search = searcher.searcher().search(new TermQuery(new Term("f", "2")), 10);
assertEquals("shard " + shard.routingEntry() + " misses new version", 1, search.totalHits);
}
}
}
}
}

2
core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
View File

@ -107,7 +107,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
Mapping mappingUpdate) {
Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
Field versionField = new NumericDocValuesField("_version", 0);
SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
document.add(uidField);
document.add(versionField);
document.add(seqID.seqNo);

2
core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View File

@ -551,7 +551,7 @@ public class IndexShardTests extends IndexShardTestCase {
ParseContext.Document document, BytesReference source, Mapping mappingUpdate) {
Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
Field versionField = new NumericDocValuesField("_version", 0);
SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
document.add(uidField);
document.add(versionField);
document.add(seqID.seqNo);

2
core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
View File

@ -332,7 +332,7 @@ public class RefreshListenersTests extends ESTestCase {
document.add(new TextField("test", testFieldValue, Field.Store.YES));
Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY);
SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
document.add(uidField);
document.add(versionField);
document.add(seqID.seqNo);

165
core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
View File

@ -36,8 +36,10 @@ import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
@ -84,6 +86,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@ -101,6 +104,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
import static org.elasticsearch.common.util.BigArrays.NON_RECYCLING_INSTANCE;
import static org.hamcrest.Matchers.containsString;
@ -124,7 +128,7 @@ public class TranslogTests extends ESTestCase {
if (translog.isOpen()) {
if (translog.currentFileGeneration() > 1) {
translog.commit();
translog.commit(translog.currentFileGeneration());
assertFileDeleted(translog, translog.currentFileGeneration() - 1);
}
translog.close();
@ -287,7 +291,7 @@ public class TranslogTests extends ESTestCase {
assertThat(snapshot, SnapshotMatchers.equalsTo(ops));
assertThat(snapshot.totalOperations(), equalTo(ops.size()));
translog.commit();
translog.commit(translog.currentFileGeneration());
snapshot = translog.newSnapshot();
assertThat(snapshot, SnapshotMatchers.size(0));
assertThat(snapshot.totalOperations(), equalTo(0));
@ -373,7 +377,7 @@ public class TranslogTests extends ESTestCase {
}
}
translog.commit();
translog.commit(translog.currentFileGeneration());
{
final TranslogStats stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(0L));
@ -446,7 +450,7 @@ public class TranslogTests extends ESTestCase {
try (Translog.View view = translog.newView()) {
Translog.Snapshot snapshot2 = translog.newSnapshot();
translog.commit();
translog.commit(translog.currentFileGeneration());
assertThat(snapshot2, SnapshotMatchers.equalsTo(ops));
assertThat(snapshot2.totalOperations(), equalTo(ops.size()));
}
@ -821,7 +825,7 @@ public class TranslogTests extends ESTestCase {
break;
}
}
translog.commit();
translog.commit(translog.currentFileGeneration());
}
} finally {
run.set(false);
@ -858,7 +862,7 @@ public class TranslogTests extends ESTestCase {
assertTrue("we only synced a previous operation yet", translog.syncNeeded());
}
if (rarely()) {
translog.commit();
translog.commit(translog.currentFileGeneration());
assertFalse("location is from a previous translog - already synced", translog.ensureSynced(location)); // not syncing now
assertFalse("no sync needed since no operations in current translog", translog.syncNeeded());
}
@ -878,7 +882,7 @@ public class TranslogTests extends ESTestCase {
ArrayList<Location> locations = new ArrayList<>();
for (int op = 0; op < translogOperations; op++) {
if (rarely()) {
translog.commit(); // do this first so that there is at least one pending tlog entry
translog.commit(translog.currentFileGeneration()); // do this first so that there is at least one pending tlog entry
}
final Translog.Location location = translog.add(new Translog.Index("test", "" + op, Integer.toString(++count).getBytes(Charset.forName("UTF-8"))));
locations.add(location);
@ -889,7 +893,7 @@ public class TranslogTests extends ESTestCase {
assertTrue("this operation has not been synced", translog.ensureSynced(locations.stream()));
assertFalse("the last call to ensureSycned synced all previous ops", translog.syncNeeded()); // we are the last location so everything should be synced
} else if (rarely()) {
translog.commit();
translog.commit(translog.currentFileGeneration());
assertFalse("location is from a previous translog - already synced", translog.ensureSynced(locations.stream())); // not syncing now
assertFalse("no sync needed since no operations in current translog", translog.syncNeeded());
} else {
@ -909,7 +913,7 @@ public class TranslogTests extends ESTestCase {
for (int op = 0; op < translogOperations; op++) {
locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(++count).getBytes(Charset.forName("UTF-8")))));
if (rarely() && translogOperations > op + 1) {
translog.commit();
translog.commit(translog.currentFileGeneration());
}
}
Collections.shuffle(locations, random());
@ -1074,7 +1078,7 @@ public class TranslogTests extends ESTestCase {
locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8")))));
final boolean commit = commitOften ? frequently() : rarely();
if (commit && op < translogOperations - 1) {
translog.commit();
translog.commit(translog.currentFileGeneration());
minUncommittedOp = op + 1;
translogGeneration = translog.getGeneration();
}
@ -1300,7 +1304,7 @@ public class TranslogTests extends ESTestCase {
for (int op = 0; op < translogOperations; op++) {
locations.add(translog.add(new Translog.Index("test", "" + op, Integer.toString(op).getBytes(Charset.forName("UTF-8")))));
if (randomBoolean()) {
translog.commit();
translog.commit(translog.currentFileGeneration());
firstUncommitted = op + 1;
}
}
@ -1483,7 +1487,7 @@ public class TranslogTests extends ESTestCase {
}
try {
translog.commit();
translog.commit(translog.currentFileGeneration());
fail("already closed");
} catch (AlreadyClosedException ex) {
assertNotNull(ex.getCause());
@ -1930,7 +1934,7 @@ public class TranslogTests extends ESTestCase {
if (randomBoolean()) {
failableTLog.prepareCommit();
}
failableTLog.commit();
failableTLog.commit(translog.currentFileGeneration());
syncedDocs.clear();
}
}
@ -2048,7 +2052,7 @@ public class TranslogTests extends ESTestCase {
public void testTranslogOpSerialization() throws Exception {
BytesReference B_1 = new BytesArray(new byte[]{1});
SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
assert Version.CURRENT.major <= 6 : "Using UNASSIGNED_SEQ_NO can be removed in 7.0, because 6.0+ nodes have actual sequence numbers";
long randomSeqNum = randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : randomNonNegativeLong();
long randomPrimaryTerm = randomBoolean() ? 0 : randomNonNegativeLong();
@ -2110,12 +2114,13 @@ public class TranslogTests extends ESTestCase {
for (int i = 0; i <= rolls; i++) {
assertFileIsPresent(translog, generation + i);
}
translog.commit();
translog.commit(generation + rolls);
assertThat(translog.currentFileGeneration(), equalTo(generation + rolls + 1));
assertThat(translog.totalOperations(), equalTo(0));
for (int i = 0; i <= rolls; i++) {
for (int i = 0; i < rolls; i++) {
assertFileDeleted(translog, generation + i);
}
assertFileIsPresent(translog, generation + rolls);
assertFileIsPresent(translog, generation + rolls + 1);
}
@ -2167,7 +2172,7 @@ public class TranslogTests extends ESTestCase {
}
}
translog.commit();
translog.commit(generation + rollsBefore + 1);
for (int i = 0; i <= rollsBefore; i++) {
assertFileDeleted(translog, generation + i);
@ -2178,4 +2183,130 @@ public class TranslogTests extends ESTestCase {
}
public void testMinGenerationForSeqNo() throws IOException {
final long initialGeneration = translog.getGeneration().translogFileGeneration;
final int operations = randomIntBetween(1, 4096);
final List<Long> shuffledSeqNos = LongStream.range(0, operations).boxed().collect(Collectors.toList());
Randomness.shuffle(shuffledSeqNos);
final List<Tuple<Long, Long>> seqNos = new ArrayList<>();
final Map<Long, Long> terms = new HashMap<>();
for (final Long seqNo : shuffledSeqNos) {
seqNos.add(Tuple.tuple(seqNo, terms.computeIfAbsent(seqNo, k -> 0L)));
Long repeatingTermSeqNo = randomFrom(seqNos.stream().map(Tuple::v1).collect(Collectors.toList()));
seqNos.add(Tuple.tuple(repeatingTermSeqNo, terms.computeIfPresent(repeatingTermSeqNo, (s, t) -> t + 1)));
}
for (final Tuple<Long, Long> tuple : seqNos) {
translog.add(new Translog.NoOp(tuple.v1(), tuple.v2(), "test"));
if (rarely()) {
translog.rollGeneration();
}
}
Map<Long, Set<Tuple<Long, Long>>> generations = new HashMap<>();
translog.commit(initialGeneration);
for (long seqNo = 0; seqNo < operations; seqNo++) {
final Set<Tuple<Long, Long>> seenSeqNos = new HashSet<>();
final long generation = translog.getMinGenerationForSeqNo(seqNo).translogFileGeneration;
for (long g = generation; g < translog.currentFileGeneration(); g++) {
if (!generations.containsKey(g)) {
final Set<Tuple<Long, Long>> generationSeenSeqNos = new HashSet<>();
final Checkpoint checkpoint = Checkpoint.read(translog.location().resolve(Translog.getCommitCheckpointFileName(g)));
try (TranslogReader reader = translog.openReader(translog.location().resolve(Translog.getFilename(g)), checkpoint)) {
Translog.Snapshot snapshot = reader.newSnapshot();
Translog.Operation operation;
while ((operation = snapshot.next()) != null) {
generationSeenSeqNos.add(Tuple.tuple(operation.seqNo(), operation.primaryTerm()));
}
}
generations.put(g, generationSeenSeqNos);
}
seenSeqNos.addAll(generations.get(g));
}
final long seqNoLowerBound = seqNo;
final Set<Tuple<Long, Long>> expected = seqNos.stream().filter(t -> t.v1() >= seqNoLowerBound).collect(Collectors.toSet());
seenSeqNos.retainAll(expected);
assertThat(seenSeqNos, equalTo(expected));
}
}
public void testSimpleCommit() throws IOException {
final int operations = randomIntBetween(1, 4096);
long seqNo = 0;
for (int i = 0; i < operations; i++) {
translog.add(new Translog.NoOp(seqNo++, 0, "test'"));
if (rarely()) {
translog.rollGeneration();
}
}
final long generation =
randomIntBetween(1, Math.toIntExact(translog.currentFileGeneration()));
translog.commit(generation);
for (long i = 0; i < generation; i++) {
assertFileDeleted(translog, i);
}
for (long i = generation; i <= translog.currentFileGeneration(); i++) {
assertFileIsPresent(translog, i);
}
}
public void testPrepareCommitAndCommit() throws IOException {
final int operations = randomIntBetween(1, 4096);
long seqNo = 0;
long last = -1;
for (int i = 0; i < operations; i++) {
translog.add(new Translog.NoOp(seqNo++, 0, "test"));
if (rarely()) {
final long generation = translog.currentFileGeneration();
translog.prepareCommit();
if (rarely()) {
// simulate generation filling up and rolling between preparing the commit and committing
translog.rollGeneration();
}
final int committedGeneration = randomIntBetween(Math.max(1, Math.toIntExact(last)), Math.toIntExact(generation));
translog.commit(committedGeneration);
last = committedGeneration;
for (long g = 0; i < generation; g++) {
assertFileDeleted(translog, g);
}
for (long g = generation; g < translog.currentFileGeneration(); g++) {
assertFileIsPresent(translog, g);
}
}
}
}
public void testCommitWithOpenView() throws IOException {
final int operations = randomIntBetween(1, 4096);
long seqNo = 0;
long lastCommittedGeneration = -1;
for (int i = 0; i < operations; i++) {
translog.add(new Translog.NoOp(seqNo++, 0, "test"));
if (rarely()) {
try (Translog.View ignored = translog.newView()) {
final long viewGeneration = lastCommittedGeneration;
translog.prepareCommit();
final long committedGeneration = randomIntBetween(
Math.max(1, Math.toIntExact(lastCommittedGeneration)),
Math.toIntExact(translog.currentFileGeneration()));
translog.commit(committedGeneration);
lastCommittedGeneration = committedGeneration;
// with an open view, committing should preserve generations back to the last committed generation
for (long g = 1; g < Math.min(lastCommittedGeneration, viewGeneration); g++) {
assertFileDeleted(translog, g);
}
// the view generation could be -1 if no commit has been performed
final long max = Math.max(1, Math.min(lastCommittedGeneration, viewGeneration));
for (long g = max; g < translog.currentFileGeneration(); g++) {
assertFileIsPresent(translog, g);
}
}
}
}
}
}

85
core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
View File

@ -91,7 +91,11 @@ public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestC
RoutingNode localRoutingNode = state.getRoutingNodes().node(state.getNodes().getLocalNodeId());
if (localRoutingNode != null) {
if (enableRandomFailures == false) {
assertThat("failed shard cache should be empty", failedShardsCache.values(), empty());
// initializing a shard should succeed when enableRandomFailures is disabled
// active shards can be failed if state persistence was disabled in an earlier CS update
if (failedShardsCache.values().stream().anyMatch(ShardRouting::initializing)) {
fail("failed shard cache should not contain initializing shard routing: " + failedShardsCache.values());
}
}
// check that all shards in local routing nodes have been allocated
for (ShardRouting shardRouting : localRoutingNode) {
@ -100,41 +104,50 @@ public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestC
MockIndexShard shard = indicesService.getShardOrNull(shardRouting.shardId());
ShardRouting failedShard = failedShardsCache.get(shardRouting.shardId());
if (enableRandomFailures) {
if (shard == null && failedShard == null) {
fail("Shard with id " + shardRouting + " expected but missing in indicesService and failedShardsCache");
if (state.blocks().disableStatePersistence()) {
if (shard != null) {
fail("Shard with id " + shardRouting + " should be removed from indicesService due to disabled state persistence");
}
} else {
if (failedShard != null && failedShard.isSameAllocation(shardRouting) == false) {
fail("Shard cache has not been properly cleaned for " + failedShard);
}
} else {
if (shard == null) {
fail("Shard with id " + shardRouting + " expected but missing in indicesService");
if (shard == null && failedShard == null) {
// shard must either be there or there must be a failure
fail("Shard with id " + shardRouting + " expected but missing in indicesService and failedShardsCache");
}
}
if (shard != null) {
AllocatedIndex<? extends Shard> indexService = indicesService.indexService(index);
assertTrue("Index " + index + " expected but missing in indicesService", indexService != null);
// index metadata has been updated
assertThat(indexService.getIndexSettings().getIndexMetaData(), equalTo(indexMetaData));
// shard has been created
if (enableRandomFailures == false || failedShard == null) {
assertTrue("Shard with id " + shardRouting + " expected but missing in indexService", shard != null);
// shard has latest shard routing
assertThat(shard.routingEntry(), equalTo(shardRouting));
if (enableRandomFailures == false) {
if (shard == null && shardRouting.initializing() && failedShard == shardRouting) {
// initializing a shard should succeed when enableRandomFailures is disabled
fail("Shard with id " + shardRouting + " expected but missing in indicesService " + failedShard);
}
}
if (shard.routingEntry().primary() && shard.routingEntry().active()) {
IndexShardRoutingTable shardRoutingTable = state.routingTable().shardRoutingTable(shard.shardId());
Set<String> activeIds = shardRoutingTable.activeShards().stream()
.map(r -> r.allocationId().getId()).collect(Collectors.toSet());
Set<String> initializingIds = shardRoutingTable.getAllInitializingShards().stream()
.map(r -> r.allocationId().getId()).collect(Collectors.toSet());
assertThat(shard.routingEntry() + " isn't updated with active aIDs", shard.activeAllocationIds, equalTo(activeIds));
assertThat(shard.routingEntry() + " isn't updated with init aIDs", shard.initializingAllocationIds,
equalTo(initializingIds));
if (shard != null) {
AllocatedIndex<? extends Shard> indexService = indicesService.indexService(index);
assertTrue("Index " + index + " expected but missing in indicesService", indexService != null);
// index metadata has been updated
assertThat(indexService.getIndexSettings().getIndexMetaData(), equalTo(indexMetaData));
// shard has been created
if (enableRandomFailures == false || failedShard == null) {
assertTrue("Shard with id " + shardRouting + " expected but missing in indexService", shard != null);
// shard has latest shard routing
assertThat(shard.routingEntry(), equalTo(shardRouting));
}
if (shard.routingEntry().primary() && shard.routingEntry().active()) {
IndexShardRoutingTable shardRoutingTable = state.routingTable().shardRoutingTable(shard.shardId());
Set<String> activeIds = shardRoutingTable.activeShards().stream()
.map(r -> r.allocationId().getId()).collect(Collectors.toSet());
Set<String> initializingIds = shardRoutingTable.getAllInitializingShards().stream()
.map(r -> r.allocationId().getId()).collect(Collectors.toSet());
assertThat(shard.routingEntry() + " isn't updated with active aIDs", shard.activeAllocationIds,
equalTo(activeIds));
assertThat(shard.routingEntry() + " isn't updated with init aIDs", shard.initializingAllocationIds,
equalTo(initializingIds));
}
}
}
}
@ -142,6 +155,10 @@ public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestC
// all other shards / indices have been cleaned up
for (AllocatedIndex<? extends Shard> indexService : indicesService) {
if (state.blocks().disableStatePersistence()) {
fail("Index service " + indexService.index() + " should be removed from indicesService due to disabled state persistence");
}
assertTrue(state.metaData().getIndexSafe(indexService.index()) != null);
boolean shardsFound = false;
@ -158,13 +175,9 @@ public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestC
}
if (shardsFound == false) {
if (enableRandomFailures) {
// check if we have shards of that index in failedShardsCache
// if yes, we might not have cleaned the index as failedShardsCache can be populated by another thread
assertFalse(failedShardsCache.keySet().stream().noneMatch(shardId -> shardId.getIndex().equals(indexService.index())));
} else {
fail("index service for index " + indexService.index() + " has no shards");
}
// check if we have shards of that index in failedShardsCache
// if yes, we might not have cleaned the index as failedShardsCache can be populated by another thread
assertFalse(failedShardsCache.keySet().stream().noneMatch(shardId -> shardId.getIndex().equals(indexService.index())));
}
}

20
core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
View File

@ -32,6 +32,8 @@ import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.action.shard.ShardStateAction;
import org.elasticsearch.cluster.block.ClusterBlock;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
@ -44,6 +46,7 @@ import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.index.Index;
import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
import org.elasticsearch.repositories.RepositoriesService;
@ -231,6 +234,23 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice
public ClusterState randomlyUpdateClusterState(ClusterState state,
Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap,
Supplier<MockIndicesService> indicesServiceSupplier) {
// randomly remove no_master blocks
if (randomBoolean() && state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)) {
state = ClusterState.builder(state).blocks(
ClusterBlocks.builder().blocks(state.blocks()).removeGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)).build();
}
// randomly add no_master blocks
if (rarely() && state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID) == false) {
ClusterBlock block = randomBoolean() ? DiscoverySettings.NO_MASTER_BLOCK_ALL : DiscoverySettings.NO_MASTER_BLOCK_WRITES;
state = ClusterState.builder(state).blocks(ClusterBlocks.builder().blocks(state.blocks()).addGlobalBlock(block)).build();
}
// if no_master block is in place, make no other cluster state changes
if (state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)) {
return state;
}
// randomly create new indices (until we have 200 max)
for (int i = 0; i < randomInt(5); i++) {
if (state.metaData().indices().size() > 200) {

2
core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
View File

@ -204,7 +204,7 @@ public class RecoverySourceHandlerTests extends ESTestCase {
document.add(new TextField("test", "test", Field.Store.YES));
final Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
final Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY);
final SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
final SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
document.add(uidField);
document.add(versionField);
document.add(seqID.seqNo);

15
core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java → core/src/test/java/org/elasticsearch/node/InternalSettingsPreparerTests.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.elasticsearch.node.internal;
package org.elasticsearch.node;
import org.elasticsearch.cli.MockTerminal;
import org.elasticsearch.cluster.ClusterName;
@ -182,18 +182,11 @@ public class InternalSettingsPreparerTests extends ESTestCase {
assertEquals("secret", fakeSetting.get(env.settings()).toString());
}
public void testDefaultProperties() throws Exception {
public void testDefaultPropertiesDoNothing() throws Exception {
Map<String, String> props = Collections.singletonMap("default.setting", "foo");
Environment env = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null, props);
assertEquals("foo", env.settings().get("setting"));
assertEquals("foo", env.settings().get("default.setting"));
assertNull(env.settings().get("setting"));
}
public void testDefaultPropertiesOverride() throws Exception {
Path configDir = homeDir.resolve("config");
Files.createDirectories(configDir);
Files.write(configDir.resolve("elasticsearch.yml"), Collections.singletonList("setting: bar"), StandardCharsets.UTF_8);
Map<String, String> props = Collections.singletonMap("default.setting", "foo");
Environment env = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null, props);
assertEquals("bar", env.settings().get("setting"));
}
}

45
core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSamplerTests.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.aggregations.bucket.sampler;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.search.aggregations.InternalAggregations;
import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregationTestCase;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import java.util.List;
import java.util.Map;
public class InternalSamplerTests extends InternalSingleBucketAggregationTestCase<InternalSampler> {
@Override
protected InternalSampler createTestInstance(String name, long docCount, InternalAggregations aggregations,
List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
return new InternalSampler(name, docCount, aggregations, pipelineAggregators, metaData);
}
@Override
protected void extraAssertReduced(InternalSampler reduced, List<InternalSampler> inputs) {
// Nothing extra to assert
}
@Override
protected Writeable.Reader<InternalSampler> instanceReader() {
return InternalSampler::new;
}
}

153
core/src/test/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorTests.java Normal file
View File

@ -0,0 +1,153 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.aggregations.metrics.geocentroid;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.LatLonDocValuesField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.index.mapper.GeoPointFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.test.geo.RandomGeoGenerator;
import java.io.IOException;
public class GeoCentroidAggregatorTests extends AggregatorTestCase {
private static final double GEOHASH_TOLERANCE = 1E-4D;
public void testEmpty() throws Exception {
try (Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg")
.field("field");
MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType();
fieldType.setHasDocValues(true);
fieldType.setName("field");
try (IndexReader reader = w.getReader()) {
IndexSearcher searcher = new IndexSearcher(reader);
InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType);
assertNull(result.centroid());
}
}
}
public void testUnmapped() throws Exception {
try (Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg")
.field("another_field");
Document document = new Document();
document.add(new LatLonDocValuesField("field", 10, 10));
w.addDocument(document);
try (IndexReader reader = w.getReader()) {
IndexSearcher searcher = new IndexSearcher(reader);
MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType();
fieldType.setHasDocValues(true);
fieldType.setName("another_field");
InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType);
assertNull(result.centroid());
fieldType = new GeoPointFieldMapper.GeoPointFieldType();
fieldType.setHasDocValues(true);
fieldType.setName("field");
result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType);
assertNull(result.centroid());
}
}
}
public void testSingleValuedField() throws Exception {
int numDocs = scaledRandomIntBetween(64, 256);
int numUniqueGeoPoints = randomIntBetween(1, numDocs);
try (Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
GeoPoint expectedCentroid = new GeoPoint(0, 0);
GeoPoint[] singleValues = new GeoPoint[numUniqueGeoPoints];
for (int i = 0 ; i < singleValues.length; i++) {
singleValues[i] = RandomGeoGenerator.randomPoint(random());
}
GeoPoint singleVal;
for (int i = 0; i < numDocs; i++) {
singleVal = singleValues[i % numUniqueGeoPoints];
Document document = new Document();
document.add(new LatLonDocValuesField("field", singleVal.getLat(), singleVal.getLon()));
w.addDocument(document);
expectedCentroid = expectedCentroid.reset(expectedCentroid.lat() + (singleVal.lat() - expectedCentroid.lat()) / (i + 1),
expectedCentroid.lon() + (singleVal.lon() - expectedCentroid.lon()) / (i + 1));
}
assertCentroid(w, expectedCentroid);
}
}
public void testMultiValuedField() throws Exception {
int numDocs = scaledRandomIntBetween(64, 256);
int numUniqueGeoPoints = randomIntBetween(1, numDocs);
try (Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
GeoPoint expectedCentroid = new GeoPoint(0, 0);
GeoPoint[] multiValues = new GeoPoint[numUniqueGeoPoints];
for (int i = 0 ; i < multiValues.length; i++) {
multiValues[i] = RandomGeoGenerator.randomPoint(random());
}
final GeoPoint[] multiVal = new GeoPoint[2];
for (int i = 0; i < numDocs; i++) {
multiVal[0] = multiValues[i % numUniqueGeoPoints];
multiVal[1] = multiValues[(i+1) % numUniqueGeoPoints];
Document document = new Document();
document.add(new LatLonDocValuesField("field", multiVal[0].getLat(), multiVal[0].getLon()));
document.add(new LatLonDocValuesField("field", multiVal[1].getLat(), multiVal[1].getLon()));
w.addDocument(document);
double newMVLat = (multiVal[0].lat() + multiVal[1].lat())/2d;
double newMVLon = (multiVal[0].lon() + multiVal[1].lon())/2d;
expectedCentroid = expectedCentroid.reset(expectedCentroid.lat() + (newMVLat - expectedCentroid.lat()) / (i + 1),
expectedCentroid.lon() + (newMVLon - expectedCentroid.lon()) / (i + 1));
}
assertCentroid(w, expectedCentroid);
}
}
private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid) throws IOException {
MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType();
fieldType.setHasDocValues(true);
fieldType.setName("field");
GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg")
.field("field");
try (IndexReader reader = w.getReader()) {
IndexSearcher searcher = new IndexSearcher(reader);
InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType);
assertEquals("my_agg", result.getName());
GeoPoint centroid = result.centroid();
assertNotNull(centroid);
assertEquals(expectedCentroid.getLat(), centroid.getLat(), GEOHASH_TOLERANCE);
assertEquals(expectedCentroid.getLon(), centroid.getLon(), GEOHASH_TOLERANCE);
}
}
}

81
core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentilesTests.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.InternalAggregationTestCase;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import java.util.List;
import java.util.Map;
public class InternalTDigestPercentilesTests extends InternalAggregationTestCase<InternalTDigestPercentiles> {
private final double[] percents = randomPercents();
@Override
protected InternalTDigestPercentiles createTestInstance(String name,
List<PipelineAggregator> pipelineAggregators,
Map<String, Object> metaData) {
boolean keyed = randomBoolean();
DocValueFormat format = DocValueFormat.RAW;
TDigestState state = new TDigestState(100);
int numValues = randomInt(10);
for (int i = 0; i < numValues; ++i) {
state.add(randomDouble() * 100);
}
assertEquals(state.centroidCount(), numValues);
return new InternalTDigestPercentiles(name, percents, state, keyed, format, pipelineAggregators, metaData);
}
@Override
protected void assertReduced(InternalTDigestPercentiles reduced, List<InternalTDigestPercentiles> inputs) {
final TDigestState expectedState = new TDigestState(reduced.state.compression());
long totalCount = 0;
for (InternalTDigestPercentiles input : inputs) {
assertArrayEquals(reduced.keys, input.keys, 0d);
expectedState.add(input.state);
totalCount += input.state.size();
}
assertEquals(totalCount, reduced.state.size());
if (totalCount > 0) {
assertEquals(expectedState.quantile(0), reduced.state.quantile(0), 0d);
assertEquals(expectedState.quantile(1), reduced.state.quantile(1), 0d);
}
}
@Override
protected Writeable.Reader<InternalTDigestPercentiles> instanceReader() {
return InternalTDigestPercentiles::new;
}
private static double[] randomPercents() {
List<Double> randomCdfValues = randomSubsetOf(randomIntBetween(1, 7), 0.01d, 0.05d, 0.25d, 0.50d, 0.75d, 0.95d, 0.99d);
double[] percents = new double[randomCdfValues.size()];
for (int i = 0; i < randomCdfValues.size(); i++) {
percents[i] = randomCdfValues.get(i);
}
return percents;
}
}

3
core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHitsTests.java
View File

@ -155,7 +155,8 @@ public class InternalTopHitsTests extends InternalAggregationTestCase<InternalTo
for (int i = 0; i < expectedHitsHits.length; i++) {
expectedHitsHits[i] = allHits.get(i).v2();
}
SearchHits expectedHits = new SearchHits(expectedHitsHits, totalHits, maxScore);
// Lucene's TopDocs initializes the maxScore to Float.NaN, if there is no maxScore
SearchHits expectedHits = new SearchHits(expectedHitsHits, totalHits, maxScore == Float.MIN_VALUE ? Float.NaN : maxScore);
assertEqualsWithErrorMessageFromXContent(expectedHits, actualHits);
}

4
core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
View File

@ -1182,13 +1182,13 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
client().prepareIndex("test", "child1", "c1").setParent("p1").setSource("c_field", "blue").get();
fail();
} catch (IllegalArgumentException e) {
assertThat(e.toString(), containsString("Can't specify parent if no parent field has been configured"));
assertThat(e.toString(), containsString("can't specify parent if no parent field has been configured"));
}
try {
client().prepareIndex("test", "child2", "c2").setParent("p1").setSource("c_field", "blue").get();
fail();
} catch (IllegalArgumentException e) {
assertThat(e.toString(), containsString("Can't specify parent if no parent field has been configured"));
assertThat(e.toString(), containsString("can't specify parent if no parent field has been configured"));
}
refresh();

55
core/src/test/java/org/elasticsearch/tasks/TaskResultTests.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.test.ESTestCase;
@ -65,7 +66,7 @@ public class TaskResultTests extends ESTestCase {
try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) {
result.toXContent(builder, ToXContent.EMPTY_PARAMS);
try (XContentBuilder shuffled = shuffleXContent(builder);
XContentParser parser = createParser(shuffled)) {
XContentParser parser = createParser(shuffled)) {
read = TaskResult.PARSER.apply(parser, null);
}
} catch (IOException e) {
@ -74,16 +75,52 @@ public class TaskResultTests extends ESTestCase {
assertEquals(result, read);
}
public void testTaskInfoIsForwardCompatible() throws IOException {
TaskInfo taskInfo = randomTaskInfo();
TaskInfo read;
try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) {
builder.startObject();
taskInfo.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
try (XContentBuilder withExtraFields = addRandomUnknownFields(builder)) {
try (XContentBuilder shuffled = shuffleXContent(withExtraFields)) {
try (XContentParser parser = createParser(shuffled)) {
read = TaskInfo.PARSER.apply(parser, null);
}
}
}
} catch (IOException e) {
throw new IOException("Error processing [" + taskInfo + "]", e);
}
assertEquals(taskInfo, read);
}
private XContentBuilder addRandomUnknownFields(XContentBuilder builder) throws IOException {
try (XContentParser parser = createParser(builder)) {
Map<String, Object> map = parser.mapOrdered();
int numberOfNewFields = randomIntBetween(2, 10);
for (int i = 0; i < numberOfNewFields; i++) {
if (randomBoolean()) {
map.put("unknown_field" + i, randomAlphaOfLength(20));
} else {
map.put("unknown_field" + i, Collections.singletonMap("inner", randomAlphaOfLength(20)));
}
}
XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType());
return xContentBuilder.map(map);
}
}
private static TaskResult randomTaskResult() throws IOException {
switch (between(0, 2)) {
case 0:
return new TaskResult(randomBoolean(), randomTaskInfo());
case 1:
return new TaskResult(randomTaskInfo(), new RuntimeException("error"));
case 2:
return new TaskResult(randomTaskInfo(), randomTaskResponse());
default:
throw new UnsupportedOperationException("Unsupported random TaskResult constructor");
case 0:
return new TaskResult(randomBoolean(), randomTaskInfo());
case 1:
return new TaskResult(randomTaskInfo(), new RuntimeException("error"));
case 2:
return new TaskResult(randomTaskInfo(), randomTaskResponse());
default:
throw new UnsupportedOperationException("Unsupported random TaskResult constructor");
}
}

4
core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java
View File

@ -325,7 +325,7 @@ public class SimpleVersioningIT extends ESIntegTestCase {
ids = new IDSource() {
@Override
public String next() {
return TestUtil.randomSimpleString(random);
return TestUtil.randomSimpleString(random, 1, 10);
}
};
break;
@ -335,7 +335,7 @@ public class SimpleVersioningIT extends ESIntegTestCase {
ids = new IDSource() {
@Override
public String next() {
return TestUtil.randomRealisticUnicodeString(random);
return TestUtil.randomRealisticUnicodeString(random, 1, 20);
}
};
break;

2
distribution/build.gradle
View File

@ -127,13 +127,13 @@ configure(distributions) {
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
project.integTest {
dependsOn project.assemble
includePackaged project.name == 'integ-test-zip'
if (project.name != 'integ-test-zip') {
mustRunAfter ':distribution:integ-test-zip:integTest'
}
}
project.integTestCluster {
dependsOn project.assemble
distribution = project.name
}

2
distribution/bwc-zip/build.gradle
View File

@ -30,7 +30,7 @@ import org.elasticsearch.gradle.LoggedExec
apply plugin: 'distribution'
File checkoutDir = file("${buildDir}/bwc/checkout-5.x")
task createClone(type: Exec) {
task createClone(type: LoggedExec) {
onlyIf { checkoutDir.exists() == false }
commandLine = ['git', 'clone', rootDir, checkoutDir]
}

18
distribution/deb/src/main/packaging/init.d/elasticsearch
View File

@ -32,10 +32,6 @@ fi
# The following variables can be overwritten in $DEFAULT
# Run Elasticsearch as this user ID and group ID
ES_USER=elasticsearch
ES_GROUP=elasticsearch
# Directory where the Elasticsearch binary distribution resides
ES_HOME=/usr/share/$NAME
@ -76,6 +72,12 @@ if [ ! -z "$CONF_FILE" ]; then
exit 1
fi
# ES_USER and ES_GROUP settings were removed
if [ ! -z "$ES_USER" ] || [ ! -z "$ES_GROUP" ]; then
echo "ES_USER and ES_GROUP settings are no longer supported. To run as a custom user/group use the archive distribution of Elasticsearch."
exit 1
fi
# Define other required variables
PID_FILE="$PID_DIR/$NAME.pid"
DAEMON=$ES_HOME/bin/elasticsearch
@ -119,10 +121,10 @@ case "$1" in
# Ensure that the PID_DIR exists (it is cleaned at OS startup time)
if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then
mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR"
mkdir -p "$PID_DIR" && chown elasticsearch:elasticsearch "$PID_DIR"
fi
if [ -n "$PID_FILE" ] && [ ! -e "$PID_FILE" ]; then
touch "$PID_FILE" && chown "$ES_USER":"$ES_GROUP" "$PID_FILE"
touch "$PID_FILE" && chown elasticsearch:elasticsearch "$PID_FILE"
fi
if [ -n "$MAX_OPEN_FILES" ]; then
@ -138,7 +140,7 @@ case "$1" in
fi
# Start Daemon
start-stop-daemon -d $ES_HOME --start --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
start-stop-daemon -d $ES_HOME --start --user elasticsearch -c elasticsearch --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
return=$?
if [ $return -eq 0 ]; then
i=0
@ -162,7 +164,7 @@ case "$1" in
if [ -f "$PID_FILE" ]; then
start-stop-daemon --stop --pidfile "$PID_FILE" \
--user "$ES_USER" \
--user elasticsearch \
--quiet \
--retry forever/TERM/20 > /dev/null
if [ $? -eq 1 ]; then

19
distribution/rpm/src/main/packaging/init.d/elasticsearch
View File

@ -32,8 +32,6 @@ if [ -f /etc/rc.d/init.d/functions ]; then
fi
# Sets the default values for elasticsearch variables used in this script
ES_USER="elasticsearch"
ES_GROUP="elasticsearch"
ES_HOME="/usr/share/elasticsearch"
MAX_OPEN_FILES=65536
MAX_MAP_COUNT=262144
@ -55,6 +53,12 @@ if [ ! -z "$CONF_FILE" ]; then
exit 1
fi
# ES_USER and ES_GROUP settings were removed
if [ ! -z "$ES_USER" ] || [ ! -z "$ES_GROUP" ]; then
echo "ES_USER and ES_GROUP settings are no longer supported. To run as a custom user/group use the archive distribution of Elasticsearch."
exit 1
fi
exec="$ES_HOME/bin/elasticsearch"
prog="elasticsearch"
pidfile="$PID_DIR/${prog}.pid"
@ -67,11 +71,6 @@ export ES_STARTUP_SLEEP_TIME
lockfile=/var/lock/subsys/$prog
# backwards compatibility for old config sysconfig files, pre 0.90.1
if [ -n $USER ] && [ -z $ES_USER ] ; then
ES_USER=$USER
fi
if [ ! -x "$exec" ]; then
echo "The elasticsearch startup script does not exists or it is not executable, tried: $exec"
exit 1
@ -106,16 +105,16 @@ start() {
# Ensure that the PID_DIR exists (it is cleaned at OS startup time)
if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then
mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR"
mkdir -p "$PID_DIR" && chown elasticsearch:elasticsearch "$PID_DIR"
fi
if [ -n "$pidfile" ] && [ ! -e "$pidfile" ]; then
touch "$pidfile" && chown "$ES_USER":"$ES_GROUP" "$pidfile"
touch "$pidfile" && chown elasticsearch:elasticsearch "$pidfile"
fi
cd $ES_HOME
echo -n $"Starting $prog: "
# if not running, start it up here, usually something like "daemon $exec"
daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR
daemon --user elasticsearch --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR
retval=$?
echo
[ $retval -eq 0 ] && touch $lockfile

8
distribution/src/main/packaging/env/elasticsearch vendored
View File

@ -32,14 +32,6 @@
# SysV init.d
#
# When executing the init script, this user will be used to run the elasticsearch service.
# The default value is 'elasticsearch' and is declared in the init.d file.
# Note that this setting is only used by the init script. If changed, make sure that
# the configured user can read and write into the data, work, plugins and log directories.
# For systemd service, the user is usually configured in file /usr/lib/systemd/system/elasticsearch.service
#ES_USER=elasticsearch
#ES_GROUP=elasticsearch
# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process
ES_STARTUP_SLEEP_TIME=5

10
distribution/src/main/packaging/scripts/postinst
View File

@ -10,10 +10,6 @@
# Sets the default values for elasticsearch variables used in this script
ES_USER="elasticsearch"
ES_GROUP="elasticsearch"
# Source the default env file
ES_ENV_FILE="${path.env}"
if [ -f "$ES_ENV_FILE" ]; then
@ -110,9 +106,9 @@ elif [ "$RESTART_ON_UPGRADE" = "true" ]; then
echo " OK"
fi
chown -R $ES_USER:$ES_GROUP /var/lib/elasticsearch
chown -R $ES_USER:$ES_GROUP /var/log/elasticsearch
chown -R root:$ES_GROUP /etc/elasticsearch
chown -R elasticsearch:elasticsearch /var/lib/elasticsearch
chown -R elasticsearch:elasticsearch /var/log/elasticsearch
chown -R root:elasticsearch /etc/elasticsearch
chmod 0750 /etc/elasticsearch
chmod 0750 /etc/elasticsearch/scripts

10
distribution/src/main/packaging/scripts/postrm
View File

@ -46,8 +46,6 @@ case "$1" in
esac
# Sets the default values for elasticsearch variables used in this script
ES_USER="elasticsearch"
ES_GROUP="elasticsearch"
LOG_DIR="/var/log/elasticsearch"
PLUGINS_DIR="/usr/share/elasticsearch/plugins"
PID_DIR="/var/run/elasticsearch"
@ -95,12 +93,12 @@ if [ "$REMOVE_DIRS" = "true" ]; then
fi
if [ "$REMOVE_USER_AND_GROUP" = "true" ]; then
if id "$ES_USER" > /dev/null 2>&1 ; then
userdel "$ES_USER"
if id elasticsearch > /dev/null 2>&1 ; then
userdel elasticsearch
fi
if getent group "$ES_GROUP" > /dev/null 2>&1 ; then
groupdel "$ES_GROUP"
if getent group elasticsearch > /dev/null 2>&1 ; then
groupdel elasticsearch
fi
fi

32
distribution/src/main/packaging/scripts/preinst
View File

@ -11,10 +11,6 @@
# Sets the default values for elasticsearch variables used in this script
ES_USER="elasticsearch"
ES_GROUP="elasticsearch"
# Source the default env file
ES_ENV_FILE="${path.env}"
if [ -f "$ES_ENV_FILE" ]; then
@ -27,22 +23,22 @@ case "$1" in
install|upgrade)
# Create elasticsearch group if not existing
if ! getent group "$ES_GROUP" > /dev/null 2>&1 ; then
echo -n "Creating $ES_GROUP group..."
addgroup --quiet --system "$ES_GROUP"
if ! getent group elasticsearch > /dev/null 2>&1 ; then
echo -n "Creating elasticsearch group..."
addgroup --quiet --system elasticsearch
echo " OK"
fi
# Create elasticsearch user if not existing
if ! id $ES_USER > /dev/null 2>&1 ; then
echo -n "Creating $ES_USER user..."
if ! id elasticsearch > /dev/null 2>&1 ; then
echo -n "Creating elasticsearch user..."
adduser --quiet \
--system \
--no-create-home \
--ingroup "$ES_GROUP" \
--ingroup elasticsearch \
--disabled-password \
--shell /bin/false \
"$ES_USER"
elasticsearch
echo " OK"
fi
;;
@ -53,21 +49,21 @@ case "$1" in
1|2)
# Create elasticsearch group if not existing
if ! getent group "$ES_GROUP" > /dev/null 2>&1 ; then
echo -n "Creating $ES_GROUP group..."
groupadd -r "$ES_GROUP"
if ! getent group elasticsearch > /dev/null 2>&1 ; then
echo -n "Creating elasticsearch group..."
groupadd -r elasticsearch
echo " OK"
fi
# Create elasticsearch user if not existing
if ! id $ES_USER > /dev/null 2>&1 ; then
echo -n "Creating $ES_USER user..."
if ! id elasticsearch > /dev/null 2>&1 ; then
echo -n "Creating elasticsearch user..."
useradd -r \
-M \
--gid "$ES_GROUP" \
--gid elasticsearch \
--shell /sbin/nologin \
--comment "elasticsearch user" \
"$ES_USER"
elasticsearch
echo " OK"
fi
;;

Some files were not shown because too many files have changed in this diff Show More