diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 82a29890b0d..26e78264534 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -106,7 +106,7 @@ public class TransportClusterStatsAction extends TransportNodesAction[]> contexts = new ArrayList<>(); + final List contexts = new ArrayList<>(); final ActionListener listener; final AtomicReference expHolder; final AtomicInteger numberOfFreedSearchContexts = new AtomicInteger(0); @@ -81,7 +81,7 @@ public class TransportClearScrollAction extends HandledTransportAction[] context = parseScrollId(parsedScrollId).getContext(); + ScrollIdForNode[] context = parseScrollId(parsedScrollId).getContext(); expectedOps += context.length; this.contexts.add(context); } @@ -114,15 +114,15 @@ public class TransportClearScrollAction extends HandledTransportAction[] context : contexts) { - for (Tuple target : context) { - final DiscoveryNode node = nodes.get(target.v1()); + for (ScrollIdForNode[] context : contexts) { + for (ScrollIdForNode target : context) { + final DiscoveryNode node = nodes.get(target.getNode()); if (node == null) { onFreedContext(false); continue; } - searchServiceTransportAction.sendFreeContext(node, target.v2(), request, new ActionListener() { + searchServiceTransportAction.sendFreeContext(node, target.getScrollId(), request, new ActionListener() { @Override public void onResponse(SearchServiceTransportAction.SearchFreeContextResponse freed) { onFreedContext(freed.isFreed()); diff --git a/core/src/main/java/org/elasticsearch/action/search/type/ParsedScrollId.java b/core/src/main/java/org/elasticsearch/action/search/type/ParsedScrollId.java index 57d5a0365ca..222727df4c1 100644 --- a/core/src/main/java/org/elasticsearch/action/search/type/ParsedScrollId.java +++ b/core/src/main/java/org/elasticsearch/action/search/type/ParsedScrollId.java @@ -19,9 +19,6 @@ package org.elasticsearch.action.search.type; -import org.elasticsearch.Version; -import org.elasticsearch.common.collect.Tuple; - import java.util.Map; /** @@ -39,11 +36,11 @@ public class ParsedScrollId { private final String type; - private final Tuple[] context; + private final ScrollIdForNode[] context; private final Map attributes; - public ParsedScrollId(String source, String type, Tuple[] context, Map attributes) { + public ParsedScrollId(String source, String type, ScrollIdForNode[] context, Map attributes) { this.source = source; this.type = type; this.context = context; @@ -58,7 +55,7 @@ public class ParsedScrollId { return type; } - public Tuple[] getContext() { + public ScrollIdForNode[] getContext() { return context; } diff --git a/core/src/main/java/org/elasticsearch/action/search/type/ScrollIdForNode.java b/core/src/main/java/org/elasticsearch/action/search/type/ScrollIdForNode.java new file mode 100644 index 00000000000..38c79c91519 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/search/type/ScrollIdForNode.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.search.type; + +public class ScrollIdForNode { + private final String node; + private final long scrollId; + + public ScrollIdForNode(String node, long scrollId) { + this.node = node; + this.scrollId = scrollId; + } + + public String getNode() { + return node; + } + + public long getScrollId() { + return scrollId; + } +} diff --git a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java index 0f8da3a8ade..bb9e85810a2 100644 --- a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java +++ b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java @@ -30,7 +30,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Base64; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.internal.InternalScrollSearchRequest; @@ -103,14 +102,14 @@ public abstract class TransportSearchHelper { throw new IllegalArgumentException("Malformed scrollId [" + scrollId + "]"); } - @SuppressWarnings({"unchecked"}) Tuple[] context = new Tuple[contextSize]; + ScrollIdForNode[] context = new ScrollIdForNode[contextSize]; for (int i = 0; i < contextSize; i++) { String element = elements[index++]; int sep = element.indexOf(':'); if (sep == -1) { throw new IllegalArgumentException("Malformed scrollId [" + scrollId + "]"); } - context[i] = new Tuple<>(element.substring(sep + 1), Long.parseLong(element.substring(0, sep))); + context[i] = new ScrollIdForNode(element.substring(sep + 1), Long.parseLong(element.substring(0, sep))); } Map attributes; int attributesSize = Integer.parseInt(elements[index++]); diff --git a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java index 1683a6bbf6d..bb2c82d8831 100644 --- a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java +++ b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryAndFetchAction.java @@ -25,7 +25,6 @@ import org.elasticsearch.action.search.*; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -115,15 +114,15 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent return; } - Tuple[] context = scrollId.getContext(); + ScrollIdForNode[] context = scrollId.getContext(); for (int i = 0; i < context.length; i++) { - Tuple target = context[i]; - DiscoveryNode node = nodes.get(target.v1()); + ScrollIdForNode target = context[i]; + DiscoveryNode node = nodes.get(target.getNode()); if (node != null) { - executePhase(i, node, target.v2()); + executePhase(i, node, target.getScrollId()); } else { if (logger.isDebugEnabled()) { - logger.debug("Node [" + target.v1() + "] not available for scroll request [" + scrollId.getSource() + "]"); + logger.debug("Node [" + target.getNode() + "] not available for scroll request [" + scrollId.getSource() + "]"); } successfulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { @@ -132,11 +131,11 @@ public class TransportSearchScrollQueryAndFetchAction extends AbstractComponent } } - for (Tuple target : scrollId.getContext()) { - DiscoveryNode node = nodes.get(target.v1()); + for (ScrollIdForNode target : scrollId.getContext()) { + DiscoveryNode node = nodes.get(target.getNode()); if (node == null) { if (logger.isDebugEnabled()) { - logger.debug("Node [" + target.v1() + "] not available for scroll request [" + scrollId.getSource() + "]"); + logger.debug("Node [" + target.getNode() + "] not available for scroll request [" + scrollId.getSource() + "]"); } successfulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { diff --git a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java index 84d631e24c2..9c7742615c4 100644 --- a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java +++ b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollQueryThenFetchAction.java @@ -26,7 +26,6 @@ import org.elasticsearch.action.search.*; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -124,15 +123,15 @@ public class TransportSearchScrollQueryThenFetchAction extends AbstractComponent } final AtomicInteger counter = new AtomicInteger(scrollId.getContext().length); - Tuple[] context = scrollId.getContext(); + ScrollIdForNode[] context = scrollId.getContext(); for (int i = 0; i < context.length; i++) { - Tuple target = context[i]; - DiscoveryNode node = nodes.get(target.v1()); + ScrollIdForNode target = context[i]; + DiscoveryNode node = nodes.get(target.getNode()); if (node != null) { - executeQueryPhase(i, counter, node, target.v2()); + executeQueryPhase(i, counter, node, target.getScrollId()); } else { if (logger.isDebugEnabled()) { - logger.debug("Node [" + target.v1() + "] not available for scroll request [" + scrollId.getSource() + "]"); + logger.debug("Node [" + target.getNode() + "] not available for scroll request [" + scrollId.getSource() + "]"); } successfulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { diff --git a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollScanAction.java b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollScanAction.java index 0be39abba14..16ab26f7d46 100644 --- a/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollScanAction.java +++ b/core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScrollScanAction.java @@ -28,7 +28,6 @@ import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -125,15 +124,15 @@ public class TransportSearchScrollScanAction extends AbstractComponent { return; } - Tuple[] context = scrollId.getContext(); + ScrollIdForNode[] context = scrollId.getContext(); for (int i = 0; i < context.length; i++) { - Tuple target = context[i]; - DiscoveryNode node = nodes.get(target.v1()); + ScrollIdForNode target = context[i]; + DiscoveryNode node = nodes.get(target.getNode()); if (node != null) { - executePhase(i, node, target.v2()); + executePhase(i, node, target.getScrollId()); } else { if (logger.isDebugEnabled()) { - logger.debug("Node [" + target.v1() + "] not available for scroll request [" + scrollId.getSource() + "]"); + logger.debug("Node [" + target.getScrollId() + "] not available for scroll request [" + scrollId.getSource() + "]"); } successfulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { @@ -142,11 +141,11 @@ public class TransportSearchScrollScanAction extends AbstractComponent { } } - for (Tuple target : scrollId.getContext()) { - DiscoveryNode node = nodes.get(target.v1()); + for (ScrollIdForNode target : scrollId.getContext()) { + DiscoveryNode node = nodes.get(target.getNode()); if (node == null) { if (logger.isDebugEnabled()) { - logger.debug("Node [" + target.v1() + "] not available for scroll request [" + scrollId.getSource() + "]"); + logger.debug("Node [" + target.getNode() + "] not available for scroll request [" + scrollId.getSource() + "]"); } successfulOps.decrementAndGet(); if (counter.decrementAndGet() == 0) { diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index a5874dc7846..06df386828e 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -78,7 +78,7 @@ public class UpdateRequest extends InstanceShardOperationRequest private boolean scriptedUpsert = false; private boolean docAsUpsert = false; - private boolean detectNoop = false; + private boolean detectNoop = true; @Nullable private IndexRequest doc; @@ -243,7 +243,7 @@ public class UpdateRequest extends InstanceShardOperationRequest * The script to execute. Note, make sure not to send different script each * times and instead use script params if possible with the same * (automatically compiled) script. - * + * * @deprecated Use {@link #script(Script)} instead */ @Deprecated @@ -256,7 +256,7 @@ public class UpdateRequest extends InstanceShardOperationRequest * The script to execute. Note, make sure not to send different script each * times and instead use script params if possible with the same * (automatically compiled) script. - * + * * @deprecated Use {@link #script(Script)} instead */ @Deprecated @@ -267,7 +267,7 @@ public class UpdateRequest extends InstanceShardOperationRequest /** * The language of the script to execute. - * + * * @deprecated Use {@link #script(Script)} instead */ @Deprecated @@ -286,7 +286,7 @@ public class UpdateRequest extends InstanceShardOperationRequest /** * Add a script parameter. - * + * * @deprecated Use {@link #script(Script)} instead */ @Deprecated @@ -311,7 +311,7 @@ public class UpdateRequest extends InstanceShardOperationRequest /** * Sets the script parameters to use with the script. - * + * * @deprecated Use {@link #script(Script)} instead */ @Deprecated @@ -338,7 +338,7 @@ public class UpdateRequest extends InstanceShardOperationRequest * The script to execute. Note, make sure not to send different script each * times and instead use script params if possible with the same * (automatically compiled) script. - * + * * @deprecated Use {@link #script(Script)} instead */ @Deprecated @@ -360,7 +360,7 @@ public class UpdateRequest extends InstanceShardOperationRequest * The script type * @param scriptParams * The script parameters - * + * * @deprecated Use {@link #script(Script)} instead */ @Deprecated @@ -623,7 +623,7 @@ public class UpdateRequest extends InstanceShardOperationRequest } /** - * Should this update attempt to detect if it is a noop? + * Should this update attempt to detect if it is a noop? Defaults to true. * @return this for chaining */ public UpdateRequest detectNoop(boolean detectNoop) { @@ -631,6 +631,9 @@ public class UpdateRequest extends InstanceShardOperationRequest return this; } + /** + * Should this update attempt to detect if it is a noop? Defaults to true. + */ public boolean detectNoop() { return detectNoop; } @@ -699,16 +702,15 @@ public class UpdateRequest extends InstanceShardOperationRequest this.docAsUpsert = shouldUpsertDoc; return this; } - + public boolean scriptedUpsert(){ return this.scriptedUpsert; } - + public UpdateRequest scriptedUpsert(boolean scriptedUpsert) { this.scriptedUpsert = scriptedUpsert; return this; } - @Override public void readFrom(StreamInput in) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index d4d0ae18963..918d2d7b935 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -308,6 +308,7 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder 0) { if (!changed) { if (sb != null) { sb.setLength(0); diff --git a/core/src/main/java/org/elasticsearch/common/util/MultiDataPathUpgrader.java b/core/src/main/java/org/elasticsearch/common/util/MultiDataPathUpgrader.java index 3425d151c34..82ed6f2bde8 100644 --- a/core/src/main/java/org/elasticsearch/common/util/MultiDataPathUpgrader.java +++ b/core/src/main/java/org/elasticsearch/common/util/MultiDataPathUpgrader.java @@ -198,7 +198,7 @@ public class MultiDataPathUpgrader { } } } - return new ShardPath(target.path, target.path, IndexMetaData.INDEX_UUID_NA_VALUE /* we don't know */, shard); + return new ShardPath(false, target.path, target.path, IndexMetaData.INDEX_UUID_NA_VALUE /* we don't know */, shard); } private ShardFileInfo[] getShardFileInfo(ShardId shard, NodeEnvironment.NodePath[] paths) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java index 7be71c89ece..f38a894cc7e 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexService.java +++ b/core/src/main/java/org/elasticsearch/index/IndexService.java @@ -22,14 +22,12 @@ package org.elasticsearch.index; import com.google.common.base.Function; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; - import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.*; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; @@ -40,7 +38,6 @@ import org.elasticsearch.index.cache.IndexCache; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.deletionpolicy.DeletionPolicyModule; import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.shard.StoreRecoveryService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.settings.IndexSettings; @@ -102,7 +99,25 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone private final NodeEnvironment nodeEnv; private final IndicesService indicesServices; - private volatile ImmutableMap> shards = ImmutableMap.of(); + private volatile ImmutableMap shards = ImmutableMap.of(); + + private static class IndexShardInjectorPair { + private final IndexShard indexShard; + private final Injector injector; + + public IndexShardInjectorPair(IndexShard indexShard, Injector injector) { + this.indexShard = indexShard; + this.injector = injector; + } + + public IndexShard getIndexShard() { + return indexShard; + } + + public Injector getInjector() { + return injector; + } + } private final AtomicBoolean closed = new AtomicBoolean(false); private final AtomicBoolean deleted = new AtomicBoolean(false); @@ -147,10 +162,10 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone @Override public Iterator iterator() { - return Iterators.transform(shards.values().iterator(), new Function, IndexShard>() { + return Iterators.transform(shards.values().iterator(), new Function() { @Override - public IndexShard apply(Tuple input) { - return input.v1(); + public IndexShard apply(IndexShardInjectorPair input) { + return input.getIndexShard(); } }); } @@ -164,9 +179,9 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone */ @Nullable public IndexShard shard(int shardId) { - Tuple indexShardInjectorTuple = shards.get(shardId); - if (indexShardInjectorTuple != null) { - return indexShardInjectorTuple.v1(); + IndexShardInjectorPair indexShardInjectorPair = shards.get(shardId); + if (indexShardInjectorPair != null) { + return indexShardInjectorPair.getIndexShard(); } return null; } @@ -244,11 +259,11 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone * Return the shard injector for the provided id, or throw an exception if there is no such shard. */ public Injector shardInjectorSafe(int shardId) { - Tuple tuple = shards.get(shardId); - if (tuple == null) { + IndexShardInjectorPair indexShardInjectorPair = shards.get(shardId); + if (indexShardInjectorPair == null) { throw new ShardNotFoundException(new ShardId(index, shardId)); } - return tuple.v2(); + return indexShardInjectorPair.getInjector(); } public String indexUUID() { @@ -286,6 +301,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone Injector shardInjector = null; try { lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5)); + indicesLifecycle.beforeIndexShardCreated(shardId, indexSettings); ShardPath path; try { path = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings); @@ -310,7 +326,6 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone throw new IndexShardAlreadyExistsException(shardId + " already exists"); } - indicesLifecycle.beforeIndexShardCreated(shardId, indexSettings); logger.debug("creating shard_id {}", shardId); // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary. final boolean canDeleteShardContent = IndexMetaData.isOnSharedFilesystem(indexSettings) == false || @@ -348,7 +363,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone indicesLifecycle.indexShardStateChanged(indexShard, null, "shard created"); indicesLifecycle.afterIndexShardCreated(indexShard); - shards = newMapBuilder(shards).put(shardId.id(), new Tuple<>(indexShard, shardInjector)).immutableMap(); + shards = newMapBuilder(shards).put(shardId.id(), new IndexShardInjectorPair(indexShard, shardInjector)).immutableMap(); success = true; return indexShard; } catch (IOException e) { @@ -374,10 +389,10 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone return; } logger.debug("[{}] closing... (reason: [{}])", shardId, reason); - HashMap> tmpShardsMap = newHashMap(shards); - Tuple tuple = tmpShardsMap.remove(shardId); - indexShard = tuple.v1(); - shardInjector = tuple.v2(); + HashMap tmpShardsMap = newHashMap(shards); + IndexShardInjectorPair indexShardInjectorPair = tmpShardsMap.remove(shardId); + indexShard = indexShardInjectorPair.getIndexShard(); + shardInjector = indexShardInjectorPair.getInjector(); shards = ImmutableMap.copyOf(tmpShardsMap); closeShardInjector(reason, sId, shardInjector, indexShard); logger.debug("[{}] closed (reason: [{}])", shardId, reason); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperForType.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperForType.java new file mode 100644 index 00000000000..1957f7e9aa5 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperForType.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +public class DocumentMapperForType { + private final DocumentMapper documentMapper; + private final Mapping mapping; + + public DocumentMapperForType(DocumentMapper documentMapper, Mapping mapping) { + this.mapping = mapping; + this.documentMapper = documentMapper; + } + + public DocumentMapper getDocumentMapper() { + return documentMapper; + } + + public Mapping getMapping() { + return mapping; + } +} diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index db2d0b20390..4300e4cc858 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -26,24 +26,18 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterators; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.queries.TermsQuery; -import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; @@ -65,12 +59,7 @@ import org.elasticsearch.script.ScriptService; import java.io.Closeable; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -395,16 +384,16 @@ public class MapperService extends AbstractIndexComponent implements Closeable { * Returns the document mapper created, including a mapping update if the * type has been dynamically created. */ - public Tuple documentMapperWithAutoCreate(String type) { + public DocumentMapperForType documentMapperWithAutoCreate(String type) { DocumentMapper mapper = mappers.get(type); if (mapper != null) { - return Tuple.tuple(mapper, null); + return new DocumentMapperForType(mapper, null); } if (!dynamic) { throw new TypeMissingException(index, type, "trying to auto create mapping, but dynamic mapping is disabled"); } mapper = parse(type, null, true); - return Tuple.tuple(mapper, mapper.mapping()); + return new DocumentMapperForType(mapper, mapper.mapping()); } /** diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java index bc834005c44..87a76689a67 100644 --- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java @@ -21,17 +21,16 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiDocValues; -import org.apache.lucene.search.*; import org.apache.lucene.search.Filter; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryWrapperFilter; import org.apache.lucene.search.join.BitDocIdSetFilter; -import org.elasticsearch.common.ParseField; import org.apache.lucene.search.join.JoinUtil; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.Version; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.XContentParser; @@ -45,8 +44,8 @@ import org.elasticsearch.index.search.child.ChildrenConstantScoreQuery; import org.elasticsearch.index.search.child.ChildrenQuery; import org.elasticsearch.index.search.child.ScoreType; import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; +import org.elasticsearch.search.fetch.innerhits.InnerHitsSubSearchContext; import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.search.internal.SubSearchContext; import java.io.IOException; @@ -82,7 +81,7 @@ public class HasChildQueryParser implements QueryParser { int maxChildren = 0; int shortCircuitParentDocSet = 8192; String queryName = null; - Tuple innerHits = null; + InnerHitsSubSearchContext innerHits = null; String currentFieldName = null; XContentParser.Token token; @@ -152,8 +151,8 @@ public class HasChildQueryParser implements QueryParser { if (innerHits != null) { ParsedQuery parsedQuery = new ParsedQuery(innerQuery, parseContext.copyNamedQueries()); - InnerHitsContext.ParentChildInnerHits parentChildInnerHits = new InnerHitsContext.ParentChildInnerHits(innerHits.v2(), parsedQuery, null, parseContext.mapperService(), childDocMapper); - String name = innerHits.v1() != null ? innerHits.v1() : childType; + InnerHitsContext.ParentChildInnerHits parentChildInnerHits = new InnerHitsContext.ParentChildInnerHits(innerHits.getSubSearchContext(), parsedQuery, null, parseContext.mapperService(), childDocMapper); + String name = innerHits.getName() != null ? innerHits.getName() : childType; parseContext.addInnerHits(name, parentChildInnerHits); } diff --git a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryParser.java index 3d3a6625c7d..67422d0aa2f 100644 --- a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryParser.java @@ -18,15 +18,10 @@ */ package org.elasticsearch.index.query; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.Query; +import org.apache.lucene.search.*; import org.elasticsearch.Version; -import org.apache.lucene.search.QueryWrapperFilter; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.XContentParser; @@ -39,7 +34,7 @@ import org.elasticsearch.index.search.child.ParentConstantScoreQuery; import org.elasticsearch.index.search.child.ParentQuery; import org.elasticsearch.index.search.child.ScoreType; import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; -import org.elasticsearch.search.internal.SubSearchContext; +import org.elasticsearch.search.fetch.innerhits.InnerHitsSubSearchContext; import java.io.IOException; import java.util.HashSet; @@ -73,7 +68,7 @@ public class HasParentQueryParser implements QueryParser { String parentType = null; boolean score = false; String queryName = null; - Tuple innerHits = null; + InnerHitsSubSearchContext innerHits = null; String currentFieldName = null; XContentParser.Token token; @@ -146,7 +141,7 @@ public class HasParentQueryParser implements QueryParser { return query; } - static Query createParentQuery(Query innerQuery, String parentType, boolean score, QueryParseContext parseContext, Tuple innerHits) throws IOException { + static Query createParentQuery(Query innerQuery, String parentType, boolean score, QueryParseContext parseContext, InnerHitsSubSearchContext innerHits) throws IOException { DocumentMapper parentDocMapper = parseContext.mapperService().documentMapper(parentType); if (parentDocMapper == null) { throw new QueryParsingException(parseContext, "[has_parent] query configured 'parent_type' [" + parentType @@ -155,8 +150,8 @@ public class HasParentQueryParser implements QueryParser { if (innerHits != null) { ParsedQuery parsedQuery = new ParsedQuery(innerQuery, parseContext.copyNamedQueries()); - InnerHitsContext.ParentChildInnerHits parentChildInnerHits = new InnerHitsContext.ParentChildInnerHits(innerHits.v2(), parsedQuery, null, parseContext.mapperService(), parentDocMapper); - String name = innerHits.v1() != null ? innerHits.v1() : parentType; + InnerHitsContext.ParentChildInnerHits parentChildInnerHits = new InnerHitsContext.ParentChildInnerHits(innerHits.getSubSearchContext(), parsedQuery, null, parseContext.mapperService(), parentDocMapper); + String name = innerHits.getName() != null ? innerHits.getName() : parentType; parseContext.addInnerHits(name, parentChildInnerHits); } diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java index 4dc71f93393..e14720bd8d1 100644 --- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java @@ -26,14 +26,13 @@ import org.apache.lucene.search.join.ToParentBlockJoinQuery; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.support.InnerHitsQueryParserHelper; import org.elasticsearch.index.query.support.NestedInnerQueryParseSupport; import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; -import org.elasticsearch.search.internal.SubSearchContext; +import org.elasticsearch.search.fetch.innerhits.InnerHitsSubSearchContext; import java.io.IOException; @@ -120,7 +119,7 @@ public class NestedQueryParser implements QueryParser { public static class ToBlockJoinQueryBuilder extends NestedInnerQueryParseSupport { private ScoreMode scoreMode; - private Tuple innerHits; + private InnerHitsSubSearchContext innerHits; public ToBlockJoinQueryBuilder(QueryParseContext parseContext) throws IOException { super(parseContext); @@ -130,7 +129,7 @@ public class NestedQueryParser implements QueryParser { this.scoreMode = scoreMode; } - public void setInnerHits(Tuple innerHits) { + public void setInnerHits(InnerHitsSubSearchContext innerHits) { this.innerHits = innerHits; } @@ -152,8 +151,8 @@ public class NestedQueryParser implements QueryParser { if (innerHits != null) { ParsedQuery parsedQuery = new ParsedQuery(innerQuery, parseContext.copyNamedQueries()); - InnerHitsContext.NestedInnerHits nestedInnerHits = new InnerHitsContext.NestedInnerHits(innerHits.v2(), parsedQuery, null, getParentObjectMapper(), nestedObjectMapper); - String name = innerHits.v1() != null ? innerHits.v1() : path; + InnerHitsContext.NestedInnerHits nestedInnerHits = new InnerHitsContext.NestedInnerHits(innerHits.getSubSearchContext(), parsedQuery, null, getParentObjectMapper(), nestedObjectMapper); + String name = innerHits.getName() != null ? innerHits.getName() : path; parseContext.addInnerHits(name, nestedInnerHits); } diff --git a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java index 56387d90751..d61fec713d2 100644 --- a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentParser; @@ -38,6 +39,8 @@ public class PrefixQueryParser implements QueryParser { public static final String NAME = "prefix"; + private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of prefix query"); + @Inject public PrefixQueryParser() { } @@ -84,7 +87,7 @@ public class PrefixQueryParser implements QueryParser { } } } else { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, NAME_FIELD)) { queryName = parser.text(); } else { fieldName = currentFieldName; diff --git a/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java index 6e4cd451ee9..355f9f2810b 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java @@ -40,6 +40,7 @@ public class RangeQueryParser implements QueryParser { public static final String NAME = "range"; private static final ParseField FIELDDATA_FIELD = new ParseField("fielddata").withAllDeprecated("[no replacement]"); + private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of range query"); @Inject public RangeQueryParser() { @@ -109,7 +110,7 @@ public class RangeQueryParser implements QueryParser { } } } else if (token.isValue()) { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, NAME_FIELD)) { queryName = parser.text(); } else if (parseContext.parseFieldMatcher().match(currentFieldName, FIELDDATA_FIELD)) { // ignore diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java index 533e3cdfe43..5844c17de3a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.util.automaton.Operations; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentParser; @@ -41,6 +42,8 @@ public class RegexpQueryParser implements QueryParser { public static final int DEFAULT_FLAGS_VALUE = RegexpFlag.ALL.value(); + private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of regexp query"); + @Inject public RegexpQueryParser() { } @@ -96,7 +99,7 @@ public class RegexpQueryParser implements QueryParser { } } } else { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, NAME_FIELD)) { queryName = parser.text(); } else { fieldName = currentFieldName; diff --git a/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java index be74053bc71..1c3876f3292 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java @@ -22,10 +22,10 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import java.io.IOException; @@ -37,6 +37,9 @@ public class TermQueryParser implements QueryParser { public static final String NAME = "term"; + private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of term query"); + private static final ParseField BOOST_FIELD = new ParseField("boost").withAllDeprecated("boost is not supported in short version of term query"); + @Inject public TermQueryParser() { } @@ -85,9 +88,9 @@ public class TermQueryParser implements QueryParser { } } } else if (token.isValue()) { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, BOOST_FIELD)) { boost = parser.floatValue(); } else { if (fieldName != null) { diff --git a/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java b/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java index ae839c41d1c..b4d3e633c3e 100644 --- a/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java +++ b/core/src/main/java/org/elasticsearch/index/query/support/InnerHitsQueryParserHelper.java @@ -19,12 +19,12 @@ package org.elasticsearch.index.query.support; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryParsingException; import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsParseElement; +import org.elasticsearch.search.fetch.innerhits.InnerHitsSubSearchContext; import org.elasticsearch.search.fetch.script.ScriptFieldsParseElement; import org.elasticsearch.search.fetch.source.FetchSourceParseElement; import org.elasticsearch.search.highlight.HighlighterParseElement; @@ -51,7 +51,7 @@ public class InnerHitsQueryParserHelper { this.fieldDataFieldsParseElement = fieldDataFieldsParseElement; } - public Tuple parse(QueryParseContext parserContext) throws IOException, QueryParsingException { + public InnerHitsSubSearchContext parse(QueryParseContext parserContext) throws IOException, QueryParsingException { String fieldName = null; XContentParser.Token token; String innerHitName = null; @@ -74,7 +74,7 @@ public class InnerHitsQueryParserHelper { } catch (Exception e) { throw new QueryParsingException(parserContext, "Failed to parse [_inner_hits]", e); } - return new Tuple<>(innerHitName, subSearchContext); + return new InnerHitsSubSearchContext(innerHitName, subSearchContext); } public static void parseCommonInnerHitOptions(XContentParser parser, XContentParser.Token token, String fieldName, SubSearchContext subSearchContext, diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index e60b046801c..10217983f40 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -21,17 +21,10 @@ package org.elasticsearch.index.shard; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; - import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.DisjunctionMaxQuery; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.MatchNoDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryCachingPolicy; -import org.apache.lucene.search.UsageTrackingQueryCachingPolicy; +import org.apache.lucene.search.*; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.ThreadInterruptedException; @@ -48,7 +41,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.logging.ESLogger; @@ -506,13 +498,13 @@ public class IndexShard extends AbstractIndexShardComponent { } } - static Engine.Create prepareCreate(Tuple docMapper, SourceToParse source, long version, VersionType versionType, Engine.Operation.Origin origin, boolean canHaveDuplicates, boolean autoGeneratedId) { + static Engine.Create prepareCreate(DocumentMapperForType docMapper, SourceToParse source, long version, VersionType versionType, Engine.Operation.Origin origin, boolean canHaveDuplicates, boolean autoGeneratedId) { long startTime = System.nanoTime(); - ParsedDocument doc = docMapper.v1().parse(source); - if (docMapper.v2() != null) { - doc.addDynamicMappingsUpdate(docMapper.v2()); + ParsedDocument doc = docMapper.getDocumentMapper().parse(source); + if (docMapper.getMapping() != null) { + doc.addDynamicMappingsUpdate(docMapper.getMapping()); } - return new Engine.Create(docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, canHaveDuplicates, autoGeneratedId); + return new Engine.Create(docMapper.getDocumentMapper().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, canHaveDuplicates, autoGeneratedId); } public void create(Engine.Create create) { @@ -540,13 +532,13 @@ public class IndexShard extends AbstractIndexShardComponent { } } - static Engine.Index prepareIndex(Tuple docMapper, SourceToParse source, long version, VersionType versionType, Engine.Operation.Origin origin, boolean canHaveDuplicates) { + static Engine.Index prepareIndex(DocumentMapperForType docMapper, SourceToParse source, long version, VersionType versionType, Engine.Operation.Origin origin, boolean canHaveDuplicates) { long startTime = System.nanoTime(); - ParsedDocument doc = docMapper.v1().parse(source); - if (docMapper.v2() != null) { - doc.addDynamicMappingsUpdate(docMapper.v2()); + ParsedDocument doc = docMapper.getDocumentMapper().parse(source); + if (docMapper.getMapping() != null) { + doc.addDynamicMappingsUpdate(docMapper.getMapping()); } - return new Engine.Index(docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, canHaveDuplicates); + return new Engine.Index(docMapper.getDocumentMapper().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, canHaveDuplicates); } /** @@ -573,7 +565,7 @@ public class IndexShard extends AbstractIndexShardComponent { public Engine.Delete prepareDelete(String type, String id, long version, VersionType versionType, Engine.Operation.Origin origin) { long startTime = System.nanoTime(); - final DocumentMapper documentMapper = docMapper(type).v1(); + final DocumentMapper documentMapper = docMapper(type).getDocumentMapper(); return new Engine.Delete(type, id, documentMapper.uidMapper().term(Uid.createUid(type, id)), version, versionType, origin, startTime, false); } @@ -1389,7 +1381,7 @@ public class IndexShard extends AbstractIndexShardComponent { return indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE); } - private Tuple docMapper(String type) { + private DocumentMapperForType docMapper(String type) { return mapperService.documentMapperWithAutoCreate(type); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java index e22659cbd53..8feed273c4f 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.shard; import org.apache.lucene.util.IOUtils; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; @@ -41,10 +42,18 @@ public final class ShardPath { private final String indexUUID; private final ShardId shardId; private final Path shardStatePath; + private final boolean isCustomDataPath; - - public ShardPath(Path path, Path shardStatePath, String indexUUID, ShardId shardId) { - this.path = path; + public ShardPath(boolean isCustomDataPath, Path dataPath, Path shardStatePath, String indexUUID, ShardId shardId) { + assert dataPath.getFileName().toString().equals(Integer.toString(shardId.id())) : "dataPath must end with the shard ID but didn't: " + dataPath.toString(); + assert shardStatePath.getFileName().toString().equals(Integer.toString(shardId.id())) : "shardStatePath must end with the shard ID but didn't: " + dataPath.toString(); + assert dataPath.getParent().getFileName().toString().equals(shardId.getIndex()) : "dataPath must end with index/shardID but didn't: " + dataPath.toString(); + assert shardStatePath.getParent().getFileName().toString().equals(shardId.getIndex()) : "shardStatePath must end with index/shardID but didn't: " + dataPath.toString(); + if (isCustomDataPath && dataPath.equals(shardStatePath)) { + throw new IllegalArgumentException("shard state path must be different to the data path when using custom data paths"); + } + this.isCustomDataPath = isCustomDataPath; + this.path = dataPath; this.indexUUID = indexUUID; this.shardId = shardId; this.shardStatePath = shardStatePath; @@ -78,6 +87,30 @@ public final class ShardPath { return shardStatePath; } + /** + * Returns the data-path root for this shard. The root is a parent of {@link #getDataPath()} without the index name + * and the shard ID. + */ + public Path getRootDataPath() { + Path noIndexShardId = getDataPath().getParent().getParent(); + return isCustomDataPath ? noIndexShardId : noIndexShardId.getParent(); // also strip the indices folder + } + + /** + * Returns the state-path root for this shard. The root is a parent of {@link #getRootStatePath()} ()} without the index name + * and the shard ID. + */ + public Path getRootStatePath() { + return getShardStatePath().getParent().getParent().getParent(); // also strip the indices folder + } + + /** + * Returns true iff the data location is a custom data location and therefore outside of the nodes configured data paths. + */ + public boolean isCustomDataPath() { + return isCustomDataPath; + } + /** * This method walks through the nodes shard paths to find the data and state path for the given shard. If multiple * directories with a valid shard state exist the one with the highest version will be used. @@ -113,7 +146,7 @@ public final class ShardPath { dataPath = statePath; } logger.debug("{} loaded data path [{}], state path [{}]", shardId, dataPath, statePath); - return new ShardPath(dataPath, statePath, indexUUID, shardId); + return new ShardPath(NodeEnvironment.hasCustomDataPath(indexSettings), dataPath, statePath, indexUUID, shardId); } } @@ -202,7 +235,7 @@ public final class ShardPath { dataPath = statePath; } - return new ShardPath(dataPath, statePath, indexUUID, shardId); + return new ShardPath(NodeEnvironment.hasCustomDataPath(indexSettings), dataPath, statePath, indexUUID, shardId); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java index 46c03de09ce..890b70996cd 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java +++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java @@ -25,7 +25,6 @@ import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.search.Queries; @@ -67,7 +66,7 @@ public class TranslogRecoveryPerformer { this.indexCache = indexCache; } - protected Tuple docMapper(String type) { + protected DocumentMapperForType docMapper(String type) { return mapperService.documentMapperWithAutoCreate(type); // protected for testing } diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/ShardTermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/ShardTermVectorsService.java index 0d107e19b46..cb34c1167d5 100644 --- a/core/src/main/java/org/elasticsearch/index/termvectors/ShardTermVectorsService.java +++ b/core/src/main/java/org/elasticsearch/index/termvectors/ShardTermVectorsService.java @@ -20,12 +20,7 @@ package org.elasticsearch.index.termvectors; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.MultiFields; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; +import org.apache.lucene.index.*; import org.apache.lucene.index.memory.MemoryIndex; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.termvectors.TermVectorsFilter; @@ -38,20 +33,13 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetField; import org.elasticsearch.index.get.GetResult; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.settings.IndexSettings; @@ -61,14 +49,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.dfs.AggregatedDfs; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; +import java.util.*; import static org.elasticsearch.index.mapper.SourceToParse.source; @@ -308,10 +289,10 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { MapperService mapperService = indexShard.mapperService(); // TODO: make parsing not dynamically create fields not in the original mapping - Tuple docMapper = mapperService.documentMapperWithAutoCreate(type); - ParsedDocument parsedDocument = docMapper.v1().parse(source(doc).index(index).type(type).flyweight(true)); - if (docMapper.v2() != null) { - parsedDocument.addDynamicMappingsUpdate(docMapper.v2()); + DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(type); + ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(source(doc).index(index).type(type).flyweight(true)); + if (docMapper.getMapping() != null) { + parsedDocument.addDynamicMappingsUpdate(docMapper.getMapping()); } if (parsedDocument.dynamicMappingsUpdate() != null) { mappingUpdatedAction.updateMappingOnMasterSynchronously(index, type, parsedDocument.dynamicMappingsUpdate()); diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java index 3fe4d9d857a..9b1913e5854 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -32,8 +32,10 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.ReleasablePagedBytesReference; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.io.stream.*; +import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.logging.ESLogger; @@ -231,7 +233,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC } })) { long latestGeneration = -1; - List> filesToUpgrade = new ArrayList<>(); + List filesToUpgrade = new ArrayList<>(); for (Path path : stream) { Matcher matcher = parseLegacyIdPattern.matcher(path.getFileName().toString()); if (matcher.matches()) { @@ -239,7 +241,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC if (generation >= translogGeneration.translogFileGeneration) { latestGeneration = Math.max(translogGeneration.translogFileGeneration, generation); } - filesToUpgrade.add(new Tuple<>(path, generation)); + filesToUpgrade.add(new PathWithGeneration(path, generation)); } else { Matcher strict_matcher = PARSE_STRICT_ID_PATTERN.matcher(path.getFileName().toString()); if (strict_matcher.matches()) { @@ -250,17 +252,17 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC if (latestGeneration < translogGeneration.translogFileGeneration) { throw new IllegalStateException("latest found translog has a lower generation that the excepcted uncommitted " + translogGeneration.translogFileGeneration + " > " + latestGeneration); } - CollectionUtil.timSort(filesToUpgrade, new Comparator>() { + CollectionUtil.timSort(filesToUpgrade, new Comparator() { @Override - public int compare(Tuple o1, Tuple o2) { - long gen1 = o1.v2(); - long gen2 = o2.v2(); + public int compare(PathWithGeneration o1, PathWithGeneration o2) { + long gen1 = o1.getGeneration(); + long gen2 = o2.getGeneration(); return Long.compare(gen1, gen2); } }); - for (Tuple pathAndGeneration : filesToUpgrade) { - final Path path = pathAndGeneration.v1(); - final long generation = pathAndGeneration.v2(); + for (PathWithGeneration pathAndGeneration : filesToUpgrade) { + final Path path = pathAndGeneration.getPath(); + final long generation = pathAndGeneration.getGeneration(); final Path target = path.resolveSibling(getFilename(generation)); logger.debug("upgrading translog copy file from {} to {}", path, target); Files.move(path, target, StandardCopyOption.ATOMIC_MOVE); @@ -1798,4 +1800,21 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC return outstandingViews.size(); } + private static class PathWithGeneration { + private final Path path; + private final long generation; + + public PathWithGeneration(Path path, long generation) { + this.path = path; + this.generation = generation; + } + + public Path getPath() { + return path; + } + + public long getGeneration() { + return generation; + } + } } diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java index 43fdb3df675..229fdbc50cf 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -20,12 +20,7 @@ package org.elasticsearch.indices; import com.google.common.base.Function; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import com.google.common.collect.*; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.IOUtils; @@ -38,14 +33,8 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.inject.CreationException; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.Injectors; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.inject.ModulesBuilder; +import org.elasticsearch.common.inject.*; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -53,12 +42,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; import org.elasticsearch.gateway.MetaDataStateFormat; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexModule; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.LocalNodeIdModule; +import org.elasticsearch.index.*; import org.elasticsearch.index.aliases.IndexAliasesServiceModule; import org.elasticsearch.index.analysis.AnalysisModule; import org.elasticsearch.index.analysis.AnalysisService; @@ -91,11 +75,7 @@ import org.elasticsearch.plugins.PluginsService; import java.io.Closeable; import java.io.IOException; import java.nio.file.Files; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -124,7 +104,26 @@ public class IndicesService extends AbstractLifecycleComponent i private final NodeEnvironment nodeEnv; private final TimeValue shardsClosedTimeout; - private volatile Map> indices = ImmutableMap.of(); + private volatile Map indices = ImmutableMap.of(); + + static class IndexServiceInjectorPair { + private final IndexService indexService; + private final Injector injector; + + public IndexServiceInjectorPair(IndexService indexService, Injector injector) { + this.indexService = indexService; + this.injector = injector; + } + + public IndexService getIndexService() { + return indexService; + } + + public Injector getInjector() { + return injector; + } + } + private final Map> pendingDeletes = new HashMap<>(); private final OldShardsStats oldShardsStats = new OldShardsStats(); @@ -229,14 +228,14 @@ public class IndicesService extends AbstractLifecycleComponent i } Map> statsByShard = Maps.newHashMap(); - for (Tuple value : indices.values()) { - IndexService indexService = value.v1(); + for (IndexServiceInjectorPair value : indices.values()) { + IndexService indexService = value.getIndexService(); for (IndexShard indexShard : indexService) { try { if (indexShard.routingEntry() == null) { continue; } - IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard, indexShard.routingEntry(), flags) }); + IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard, flags) }); if (!statsByShard.containsKey(indexService.index())) { statsByShard.put(indexService.index(), Lists.newArrayList(indexShardStats)); } else { @@ -261,10 +260,10 @@ public class IndicesService extends AbstractLifecycleComponent i @Override public Iterator iterator() { - return Iterators.transform(indices.values().iterator(), new Function, IndexService>() { + return Iterators.transform(indices.values().iterator(), new Function() { @Override - public IndexService apply(Tuple input) { - return input.v1(); + public IndexService apply(IndexServiceInjectorPair input) { + return input.getIndexService(); } }); } @@ -279,11 +278,11 @@ public class IndicesService extends AbstractLifecycleComponent i */ @Nullable public IndexService indexService(String index) { - Tuple indexServiceInjectorTuple = indices.get(index); - if (indexServiceInjectorTuple == null) { + IndexServiceInjectorPair indexServiceInjectorPair = indices.get(index); + if (indexServiceInjectorPair == null) { return null; } else { - return indexServiceInjectorTuple.v1(); + return indexServiceInjectorPair.getIndexService(); } } @@ -352,7 +351,7 @@ public class IndicesService extends AbstractLifecycleComponent i indicesLifecycle.afterIndexCreated(indexService); - indices = newMapBuilder(indices).put(index.name(), new Tuple<>(indexService, indexInjector)).immutableMap(); + indices = newMapBuilder(indices).put(index.name(), new IndexServiceInjectorPair(indexService, indexInjector)).immutableMap(); return indexService; } @@ -377,10 +376,10 @@ public class IndicesService extends AbstractLifecycleComponent i } logger.debug("[{}] closing ... (reason [{}])", index, reason); - Map> tmpMap = newHashMap(indices); - Tuple remove = tmpMap.remove(index); - indexService = remove.v1(); - indexInjector = remove.v2(); + Map tmpMap = newHashMap(indices); + IndexServiceInjectorPair remove = tmpMap.remove(index); + indexService = remove.getIndexService(); + indexInjector = remove.getInjector(); indices = ImmutableMap.copyOf(tmpMap); } @@ -488,7 +487,7 @@ public class IndicesService extends AbstractLifecycleComponent i synchronized (this) { String indexName = metaData.index(); if (indices.containsKey(indexName)) { - String localUUid = indices.get(indexName).v1().indexUUID(); + String localUUid = indices.get(indexName).getIndexService().indexUUID(); throw new IllegalStateException("Can't delete index store for [" + indexName + "] - it's still part of the indices service [" + localUUid + "] [" + metaData.getIndexUUID() + "]"); } if (clusterState.metaData().hasIndex(indexName) && (clusterState.nodes().localNode().masterNode() == true)) { @@ -589,9 +588,9 @@ public class IndicesService extends AbstractLifecycleComponent i * @return true if the index can be deleted on this node */ public boolean canDeleteIndexContents(Index index, Settings indexSettings) { - final Tuple indexServiceInjectorTuple = this.indices.get(index.name()); + final IndexServiceInjectorPair indexServiceInjectorPair = this.indices.get(index.name()); if (IndexMetaData.isOnSharedFilesystem(indexSettings) == false) { - if (indexServiceInjectorTuple == null && nodeEnv.hasNodeFile()) { + if (indexServiceInjectorPair == null && nodeEnv.hasNodeFile()) { return true; } } else { @@ -622,10 +621,10 @@ public class IndicesService extends AbstractLifecycleComponent i } private boolean canDeleteShardContent(ShardId shardId, @IndexSettings Settings indexSettings) { - final Tuple indexServiceInjectorTuple = this.indices.get(shardId.getIndex()); + final IndexServiceInjectorPair indexServiceInjectorPair = this.indices.get(shardId.getIndex()); if (IndexMetaData.isOnSharedFilesystem(indexSettings) == false) { - if (indexServiceInjectorTuple != null && nodeEnv.hasNodeFile()) { - final IndexService indexService = indexServiceInjectorTuple.v1(); + if (indexServiceInjectorPair != null && nodeEnv.hasNodeFile()) { + final IndexService indexService = indexServiceInjectorPair.getIndexService(); return indexService.hasShard(shardId.id()) == false; } else if (nodeEnv.hasNodeFile()) { if (NodeEnvironment.hasCustomDataPath(indexSettings)) { diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java b/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java index e92563723c6..2781efc9309 100644 --- a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java +++ b/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java @@ -38,7 +38,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -275,10 +274,10 @@ public class PercolatorService extends AbstractComponent { } MapperService mapperService = documentIndexService.mapperService(); - Tuple docMapper = mapperService.documentMapperWithAutoCreate(request.documentType()); - doc = docMapper.v1().parse(source(parser).index(index).type(request.documentType()).flyweight(true)); - if (docMapper.v2() != null) { - doc.addDynamicMappingsUpdate(docMapper.v2()); + DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(request.documentType()); + doc = docMapper.getDocumentMapper().parse(source(parser).index(index).type(request.documentType()).flyweight(true)); + if (docMapper.getMapping() != null) { + doc.addDynamicMappingsUpdate(docMapper.getMapping()); } if (doc.dynamicMappingsUpdate() != null) { mappingUpdatedAction.updateMappingOnMasterSynchronously(request.shardId().getIndex(), request.documentType(), doc.dynamicMappingsUpdate()); @@ -384,8 +383,8 @@ public class PercolatorService extends AbstractComponent { try { parser = XContentFactory.xContent(fetchedDoc).createParser(fetchedDoc); MapperService mapperService = documentIndexService.mapperService(); - Tuple docMapper = mapperService.documentMapperWithAutoCreate(type); - doc = docMapper.v1().parse(source(parser).index(index).type(type).flyweight(true)); + DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(type); + doc = docMapper.getDocumentMapper().parse(source(parser).index(index).type(type).flyweight(true)); if (context.highlight() != null) { doc.setSource(fetchedDoc); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsSubSearchContext.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsSubSearchContext.java new file mode 100644 index 00000000000..35ab2e1bba7 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsSubSearchContext.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.fetch.innerhits; + +import org.elasticsearch.search.internal.SubSearchContext; + +public class InnerHitsSubSearchContext { + private final String name; + private final SubSearchContext subSearchContext; + + public InnerHitsSubSearchContext(String name, SubSearchContext subSearchContext) { + this.name = name; + this.subSearchContext = subSearchContext; + } + + public String getName() { + return name; + } + + public SubSearchContext getSubSearchContext() { + return subSearchContext; + } +} diff --git a/core/src/test/java/org/elasticsearch/common/StringsTests.java b/core/src/test/java/org/elasticsearch/common/StringsTests.java index 584428541e4..07465acaaae 100644 --- a/core/src/test/java/org/elasticsearch/common/StringsTests.java +++ b/core/src/test/java/org/elasticsearch/common/StringsTests.java @@ -29,6 +29,8 @@ public class StringsTests extends ESTestCase { assertEquals("fooBar", Strings.toCamelCase("foo_bar")); assertEquals("fooBarFooBar", Strings.toCamelCase("foo_bar_foo_bar")); assertEquals("fooBar", Strings.toCamelCase("foo_bar_")); + assertEquals("_foo", Strings.toCamelCase("_foo")); + assertEquals("_fooBar", Strings.toCamelCase("_foo_bar_")); } public void testSubstring() { diff --git a/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java b/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java index fc05a79423d..f91697884c0 100644 --- a/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java @@ -83,7 +83,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase { ShardStateMetaData.FORMAT.write(new ShardStateMetaData(metaStateVersion, true, uuid), metaStateVersion, shardDataPaths); } final Path path = randomFrom(shardDataPaths); - ShardPath targetPath = new ShardPath(path, path, uuid, new ShardId("foo", 0)); + ShardPath targetPath = new ShardPath(false, path, path, uuid, new ShardId("foo", 0)); MultiDataPathUpgrader helper = new MultiDataPathUpgrader(nodeEnvironment); helper.upgrade(shardId, targetPath); assertFalse(helper.needsUpgrading(shardId)); @@ -177,7 +177,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase { } logger.info("--> injecting index [{}] into multiple data paths", indexName); OldIndexBackwardsCompatibilityIT.copyIndex(logger, src, indexName, multiDataPath); - final ShardPath shardPath = new ShardPath(nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], IndexMetaData.INDEX_UUID_NA_VALUE, new ShardId(indexName, 0)); + final ShardPath shardPath = new ShardPath(false, nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], IndexMetaData.INDEX_UUID_NA_VALUE, new ShardId(indexName, 0)); logger.info("{}", FileSystemUtils.files(shardPath.resolveIndex())); diff --git a/core/src/test/java/org/elasticsearch/document/BulkIT.java b/core/src/test/java/org/elasticsearch/document/BulkIT.java index 3c13b3fe31e..78746847840 100644 --- a/core/src/test/java/org/elasticsearch/document/BulkIT.java +++ b/core/src/test/java/org/elasticsearch/document/BulkIT.java @@ -195,7 +195,7 @@ public class BulkIT extends ESIntegTestCase { bulkResponse = client().prepareBulk() .add(client().prepareUpdate("test", "type", "e1").setDoc("field", "2").setVersion(10)) // INTERNAL .add(client().prepareUpdate("test", "type", "e1").setDoc("field", "3").setVersion(20).setVersionType(VersionType.FORCE)) - .add(client().prepareUpdate("test", "type", "e1").setDoc("field", "3").setVersion(20).setVersionType(VersionType.INTERNAL)).get(); + .add(client().prepareUpdate("test", "type", "e1").setDoc("field", "4").setVersion(20).setVersionType(VersionType.INTERNAL)).get(); assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict")); assertThat(((UpdateResponse) bulkResponse.getItems()[1].getResponse()).getVersion(), equalTo(20l)); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java index 96239e0051b..03986188c6c 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java @@ -54,9 +54,7 @@ import java.nio.file.Path; import java.util.Set; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.*; /** * @@ -249,7 +247,7 @@ public class AnalysisModuleTests extends ESTestCase { fail("This should fail with IllegalArgumentException because the analyzers name starts with _"); } catch (ProvisionException e) { assertTrue(e.getCause() instanceof IllegalArgumentException); - assertThat(e.getCause().getMessage(), equalTo("analyzer name must not start with '_'. got \"_invalid_name\"")); + assertThat(e.getCause().getMessage(), either(equalTo("analyzer name must not start with '_'. got \"_invalid_name\"")).or(equalTo("analyzer name must not start with '_'. got \"_invalidName\""))); } } diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 8edc47410a3..f3d45a8061c 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -29,15 +29,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexDeletionPolicy; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LiveIndexWriterConfig; -import org.apache.lucene.index.LogByteSizeMergePolicy; -import org.apache.lucene.index.MergePolicy; -import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.index.*; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TermQuery; @@ -55,7 +47,6 @@ import org.elasticsearch.common.Base64; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.uid.Versions; @@ -69,16 +60,9 @@ import org.elasticsearch.index.deletionpolicy.KeepOnlyLastDeletionPolicy; import org.elasticsearch.index.deletionpolicy.SnapshotDeletionPolicy; import org.elasticsearch.index.engine.Engine.Searcher; import org.elasticsearch.index.indexing.ShardIndexingService; -import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.Mapper.BuilderContext; -import org.elasticsearch.index.mapper.MapperBuilders; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext.Document; -import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; @@ -107,13 +91,7 @@ import java.nio.charset.Charset; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; +import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; @@ -121,12 +99,7 @@ import java.util.regex.Pattern; import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY; import static org.elasticsearch.index.engine.Engine.Operation.Origin.REPLICA; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; public class InternalEngineTests extends ESTestCase { @@ -1984,8 +1957,8 @@ public class InternalEngineTests extends ESTestCase { } @Override - protected Tuple docMapper(String type) { - return new Tuple<>(docMapper, mappingUpdate); + protected DocumentMapperForType docMapper(String type) { + return new DocumentMapperForType(docMapper, mappingUpdate); } @Override diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java index 02841f7f570..f989467c0f2 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java @@ -19,16 +19,11 @@ package org.elasticsearch.index.mapper; import com.google.common.collect.ImmutableMap; - import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.*; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.core.IntegerFieldMapper; import org.elasticsearch.index.mapper.core.StringFieldMapper; @@ -368,7 +363,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { // Even if the dynamic type of our new field is long, we already have a mapping for the same field // of type string so it should be mapped as a string - DocumentMapper newMapper = indexService.mapperService().documentMapperWithAutoCreate("type2").v1(); + DocumentMapper newMapper = indexService.mapperService().documentMapperWithAutoCreate("type2").getDocumentMapper(); Mapper update = parse(newMapper, indexService.mapperService().documentMapperParser(), XContentFactory.jsonBuilder().startObject().field("my_field1", 42).endObject()); Mapper myField1Mapper = null; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java index f6a4c30c863..4ec0ff5211e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java @@ -38,7 +38,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; public class DefaultSourceMappingTests extends ESSingleNodeTestCase { @@ -202,7 +202,7 @@ public class DefaultSourceMappingTests extends ESSingleNodeTestCase { MapperService mapperService = createIndex("test").mapperService(); mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedXContent(defaultMapping), true, false); - DocumentMapper mapper = mapperService.documentMapperWithAutoCreate("my_type").v1(); + DocumentMapper mapper = mapperService.documentMapperWithAutoCreate("my_type").getDocumentMapper(); assertThat(mapper.type(), equalTo("my_type")); assertThat(mapper.sourceMapper().enabled(), equalTo(false)); } diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 1efe8d7f776..3ec731156be 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -20,9 +20,12 @@ package org.elasticsearch.index.shard; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.Constants; import org.apache.lucene.util.IOUtils; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.*; @@ -31,11 +34,16 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.query.QueryParsingException; @@ -539,4 +547,36 @@ public class IndexShardTests extends ESSingleNodeTestCase { assertPathHasBeenCleared(startDir.toAbsolutePath().toString()); assertPathHasBeenCleared(endDir.toAbsolutePath().toString()); } + + public void testShardStats() throws IOException { + createIndex("test"); + ensureGreen(); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService test = indicesService.indexService("test"); + IndexShard shard = test.shard(0); + ShardStats stats = new ShardStats(shard, new CommonStatsFlags()); + assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath()); + assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath()); + assertEquals(shard.shardPath().isCustomDataPath(), stats.isCustomDataPath()); + + if (randomBoolean() || true) { // try to serialize it to ensure values survive the serialization + BytesStreamOutput out = new BytesStreamOutput(); + stats.writeTo(out); + StreamInput in = StreamInput.wrap(out.bytes()); + stats = ShardStats.readShardStats(in); + } + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + stats.toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + String xContent = builder.string(); + StringBuilder expectedSubSequence = new StringBuilder("\"shard_path\":{\"state_path\":\""); + expectedSubSequence.append(shard.shardPath().getRootStatePath().toString()); + expectedSubSequence.append("\",\"data_path\":\""); + expectedSubSequence.append(shard.shardPath().getRootDataPath().toString()); + expectedSubSequence.append("\",\"is_custom_data_path\":").append(shard.shardPath().isCustomDataPath()).append("}"); + assumeFalse("Some path weirdness on windows", Constants.WINDOWS); + assertTrue(xContent.contains(expectedSubSequence)); + } + } diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java index e03698f62ee..32d20190890 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.shard; +import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; @@ -26,6 +27,7 @@ import org.junit.Test; import java.io.IOException; import java.nio.file.Path; +import java.util.Set; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -78,4 +80,71 @@ public class ShardPathTests extends ESTestCase { } } + @Test(expected = IllegalArgumentException.class) + public void testIllegalCustomDataPath() { + final Path path = createTempDir().resolve("foo").resolve("0"); + new ShardPath(true, path, path, "foo", new ShardId("foo", 0)); + } + + public void testValidCtor() { + final Path path = createTempDir().resolve("foo").resolve("0"); + ShardPath shardPath = new ShardPath(false, path, path, "foo", new ShardId("foo", 0)); + assertFalse(shardPath.isCustomDataPath()); + assertEquals(shardPath.getDataPath(), path); + assertEquals(shardPath.getShardStatePath(), path); + } + + public void testGetRootPaths() throws IOException { + boolean useCustomDataPath = randomBoolean(); + final Settings indexSetttings; + final Settings nodeSettings; + Settings.Builder indexSettingsBuilder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF"); + final Path customPath; + if (useCustomDataPath) { + final Path path = createTempDir(); + final boolean includeNodeId = randomBoolean(); + indexSetttings = indexSettingsBuilder.put(IndexMetaData.SETTING_DATA_PATH, "custom").build(); + nodeSettings = settingsBuilder().put("path.shared_data", path.toAbsolutePath().toAbsolutePath()) + .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH, includeNodeId).build(); + if (includeNodeId) { + customPath = path.resolve("custom").resolve("0"); + } else { + customPath = path.resolve("custom"); + } + } else { + customPath = null; + indexSetttings = indexSettingsBuilder.build(); + nodeSettings = Settings.EMPTY; + } + try (final NodeEnvironment env = newNodeEnvironment(nodeSettings)) { + ShardId shardId = new ShardId("foo", 0); + Path[] paths = env.availableShardPaths(shardId); + Path path = randomFrom(paths); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF"), 2, path); + ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, indexSetttings); + boolean found = false; + for (Path p : env.nodeDataPaths()) { + if (p.equals(shardPath.getRootStatePath())) { + found = true; + break; + } + } + assertTrue("root state paths must be a node path but wasn't: " + shardPath.getRootStatePath(), found); + found = false; + if (useCustomDataPath) { + assertNotEquals(shardPath.getRootDataPath(), shardPath.getRootStatePath()); + assertEquals(customPath, shardPath.getRootDataPath()); + } else { + assertNull(customPath); + for (Path p : env.nodeDataPaths()) { + if (p.equals(shardPath.getRootDataPath())) { + found = true; + break; + } + } + assertTrue("root state paths must be a node path but wasn't: " + shardPath.getRootDataPath(), found); + } + } + } + } diff --git a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java index 75237921fdc..e921f95394c 100644 --- a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java +++ b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java @@ -35,11 +35,11 @@ import java.util.Locale; public class IndexStoreTests extends ESTestCase { public void testStoreDirectory() throws IOException { - final Path tempDir = createTempDir(); + final Path tempDir = createTempDir().resolve("foo").resolve("0"); final IndexStoreModule.Type[] values = IndexStoreModule.Type.values(); final IndexStoreModule.Type type = RandomPicks.randomFrom(random(), values); Settings settings = Settings.settingsBuilder().put(IndexStoreModule.STORE_TYPE, type.name().toLowerCase(Locale.ROOT)).build(); - FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(tempDir, tempDir, "foo", new ShardId("foo", 0))); + FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0))); try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { switch (type) { case NIOFS: @@ -70,9 +70,9 @@ public class IndexStoreTests extends ESTestCase { } public void testStoreDirectoryDefault() throws IOException { - final Path tempDir = createTempDir(); + final Path tempDir = createTempDir().resolve("foo").resolve("0"); Settings settings = Settings.EMPTY; - FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(tempDir, tempDir, "foo", new ShardId("foo", 0))); + FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0))); try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { if (Constants.WINDOWS) { assertTrue(directory.toString(), directory instanceof MMapDirectory || directory instanceof SimpleFSDirectory); diff --git a/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerIT.java b/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerIT.java index 6179af72d34..695845bca0d 100644 --- a/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerIT.java +++ b/core/src/test/java/org/elasticsearch/indices/warmer/SimpleIndicesWarmerIT.java @@ -173,6 +173,18 @@ public class SimpleIndicesWarmerIT extends ESIntegTestCase { } } + @Test // issue 8991 + public void deleteAllIndexWarmerDoesNotThrowWhenNoWarmers() { + createIndex("test"); + DeleteWarmerResponse deleteWarmerResponse = client().admin().indices().prepareDeleteWarmer() + .setIndices("test").setNames("_all").execute().actionGet(); + assertThat(deleteWarmerResponse.isAcknowledged(), equalTo(true)); + + deleteWarmerResponse = client().admin().indices().prepareDeleteWarmer() + .setIndices("test").setNames("foo", "_all", "bar").execute().actionGet(); + assertThat(deleteWarmerResponse.isAcknowledged(), equalTo(true)); + } + @Test public void deleteIndexWarmerTest() { createIndex("test"); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java index 169901aad59..c729c2f2d04 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java @@ -255,9 +255,15 @@ public class ChildrenIT extends ESIntegTestCase { assertThat(count.getValue(), equalTo(4.)); String idToUpdate = Integer.toString(randomInt(3)); + /* + * The whole point of this test is to test these things with deleted + * docs in the index so we turn off detect_noop to make sure that + * the updates cause that. + */ UpdateResponse updateResponse = client().prepareUpdate(indexName, "child", idToUpdate) .setParent("1") .setDoc("count", 1) + .setDetectNoop(false) .get(); assertThat(updateResponse.getVersion(), greaterThan(1l)); refresh(); diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java index 6c4bd969955..ea151fcedf4 100644 --- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java +++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java @@ -20,17 +20,21 @@ package org.elasticsearch.ttl; import com.google.common.base.Predicate; + import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.update.UpdateRequestBuilder; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.get.GetField; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; +import org.elasticsearch.test.ESIntegTestCase.Scope; import java.io.IOException; import java.util.Locale; @@ -39,9 +43,17 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; @ClusterScope(scope= Scope.SUITE, numDataNodes = 1) public class SimpleTTLIT extends ESIntegTestCase { @@ -63,7 +75,6 @@ public class SimpleTTLIT extends ESIntegTestCase { .build(); } - @Test public void testSimpleTTL() throws Exception { assertAcked(prepareCreate("test") .addMapping("type1", XContentFactory.jsonBuilder() @@ -200,7 +211,7 @@ public class SimpleTTLIT extends ESIntegTestCase { assertThat(getResponse.isExists(), equalTo(false)); } - @Test // issue 5053 + // issue 5053 public void testThatUpdatingMappingShouldNotRemoveTTLConfiguration() throws Exception { String index = "foo"; String type = "mytype"; @@ -220,6 +231,63 @@ public class SimpleTTLIT extends ESIntegTestCase { assertTTLMappingEnabled(index, type); } + /** + * Test that updates with detect_noop set to true (the default) that don't + * change the source don't change the ttl. This is unexpected behavior and + * documented in ttl-field.asciidoc. If this behavior changes it is safe to + * rewrite this test to reflect the new behavior and to change the + * documentation. + */ + public void testNoopUpdate() throws IOException { + assertAcked(prepareCreate("test") + .addMapping("type1", XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("_timestamp").field("enabled", true).endObject() + .startObject("_ttl").field("enabled", true).endObject() + .endObject() + .endObject())); + ensureYellow("test"); + + long aLongTime = 10000000; + long firstTtl = aLongTime * 3; + long secondTtl = aLongTime * 2; + long thirdTtl = aLongTime * 1; + IndexResponse indexResponse = client().prepareIndex("test", "type1", "1").setSource("field1", "value1") + .setTTL(firstTtl).setRefresh(true).get(); + assertTrue(indexResponse.isCreated()); + assertThat(getTtl("type1", 1), both(lessThan(firstTtl)).and(greaterThan(secondTtl))); + + // Updating with the default detect_noop without a change to the document doesn't change the ttl. + UpdateRequestBuilder update = client().prepareUpdate("test", "type1", "1").setDoc("field1", "value1").setTtl(secondTtl); + assertThat(updateAndGetTtl(update), both(lessThan(firstTtl)).and(greaterThan(secondTtl))); + + // Updating with the default detect_noop with a change to the document does change the ttl. + update = client().prepareUpdate("test", "type1", "1").setDoc("field1", "value2").setTtl(secondTtl); + assertThat(updateAndGetTtl(update), both(lessThan(secondTtl)).and(greaterThan(thirdTtl))); + + // Updating with detect_noop=true without a change to the document doesn't change the ttl. + update = client().prepareUpdate("test", "type1", "1").setDoc("field1", "value2").setTtl(secondTtl).setDetectNoop(true); + assertThat(updateAndGetTtl(update), both(lessThan(secondTtl)).and(greaterThan(thirdTtl))); + + // Updating with detect_noop=false without a change to the document does change the ttl. + update = client().prepareUpdate("test", "type1", "1").setDoc("field1", "value2").setTtl(thirdTtl).setDetectNoop(false); + assertThat(updateAndGetTtl(update), lessThan(thirdTtl)); + } + + private long updateAndGetTtl(UpdateRequestBuilder update) { + UpdateResponse updateResponse = update.setFields("_ttl").get(); + assertThat(updateResponse.getShardInfo().getFailed(), equalTo(0)); + // You can't actually fetch _ttl from an update so we use a get. + return getTtl(updateResponse.getType(), updateResponse.getId()); + } + + private long getTtl(String type, Object id) { + GetResponse getResponse = client().prepareGet("test", type, id.toString()).setFields("_ttl").setRealtime(true).execute() + .actionGet(); + return ((Number) getResponse.getField("_ttl").getValue()).longValue(); + } + private void assertTTLMappingEnabled(String index, String type) throws IOException { String errMsg = String.format(Locale.ROOT, "Expected ttl field mapping to be enabled for %s/%s", index, type); diff --git a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java index 3f0b1bcc1ec..499fb52a98b 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.update; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -44,8 +45,10 @@ public class UpdateNoopIT extends ESIntegTestCase { updateAndCheckSource(4, fields("bar", null)); updateAndCheckSource(4, fields("bar", null)); updateAndCheckSource(5, fields("bar", "foo")); + // detect_noop defaults to true + updateAndCheckSource(5, null, fields("bar", "foo")); - assertEquals(3, totalNoopUpdates()); + assertEquals(4, totalNoopUpdates()); } @Test @@ -210,7 +213,8 @@ public class UpdateNoopIT extends ESIntegTestCase { } /** - * Totally empty requests are noop if and only if detect noops is true. + * Totally empty requests are noop if and only if detect noops is true and + * its true by default. */ @Test public void totallyEmpty() throws Exception { @@ -223,6 +227,7 @@ public class UpdateNoopIT extends ESIntegTestCase { .endObject()); update(true, 1, XContentFactory.jsonBuilder().startObject().endObject()); update(false, 2, XContentFactory.jsonBuilder().startObject().endObject()); + update(null, 2, XContentFactory.jsonBuilder().startObject().endObject()); } private XContentBuilder fields(Object... fields) throws IOException { @@ -237,17 +242,23 @@ public class UpdateNoopIT extends ESIntegTestCase { } private void updateAndCheckSource(long expectedVersion, XContentBuilder xContentBuilder) { - UpdateResponse updateResponse = update(true, expectedVersion, xContentBuilder); + updateAndCheckSource(expectedVersion, true, xContentBuilder); + } + + private void updateAndCheckSource(long expectedVersion, Boolean detectNoop, XContentBuilder xContentBuilder) { + UpdateResponse updateResponse = update(detectNoop, expectedVersion, xContentBuilder); assertEquals(updateResponse.getGetResult().sourceRef().toUtf8(), xContentBuilder.bytes().toUtf8()); } - private UpdateResponse update(boolean detectNoop, long expectedVersion, XContentBuilder xContentBuilder) { - UpdateResponse updateResponse = client().prepareUpdate("test", "type1", "1") + private UpdateResponse update(Boolean detectNoop, long expectedVersion, XContentBuilder xContentBuilder) { + UpdateRequestBuilder updateRequest = client().prepareUpdate("test", "type1", "1") .setDoc(xContentBuilder.bytes().toUtf8()) .setDocAsUpsert(true) - .setDetectNoop(detectNoop) - .setFields("_source") - .execute().actionGet(); + .setFields("_source"); + if (detectNoop != null) { + updateRequest.setDetectNoop(detectNoop); + } + UpdateResponse updateResponse = updateRequest.get(); assertThat(updateResponse.getGetResult(), notNullValue()); assertEquals(expectedVersion, updateResponse.getVersion()); return updateResponse; diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index 06958b4e624..7d2dbfaeede 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -114,25 +114,23 @@ If both `doc` and `script` is specified, then `doc` is ignored. Best is to put your field pairs of the partial document in the script itself. [float] -=== Detecting noop - -By default if `doc` is specified then the document is always updated even -if the merging process doesn't cause any changes. Specifying `detect_noop` -as `true` will cause Elasticsearch to check if there are changes and, if -there aren't, turn the update request into a noop. For example: - +=== Detecting noop updates +If `doc` is specified its value is merged with the existing `_source`. By +default the document is only reindexed if the new `_source` field differs from +the old. Setting `detect_noop` to `false` will cause Elasticsearch to always +update the document even if it hasn't changed. For example: [source,js] -------------------------------------------------- curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{ "doc" : { "name" : "new_name" }, - "detect_noop": true + "detect_noop": false }' -------------------------------------------------- -If `name` was `new_name` before the request was sent then the entire update -request is ignored. +If `name` was `new_name` before the request was sent then document is still +reindexed. [[upserts]] [float] diff --git a/docs/reference/mapping/fields/ttl-field.asciidoc b/docs/reference/mapping/fields/ttl-field.asciidoc index 26bc3e74417..07ce8a86b9e 100644 --- a/docs/reference/mapping/fields/ttl-field.asciidoc +++ b/docs/reference/mapping/fields/ttl-field.asciidoc @@ -104,3 +104,7 @@ may still be retrieved before they are purged. How many deletions are handled by a single <> request. The default value is `10000`. +==== Note on `detect_noop` +If an update tries to update just the `_ttl` without changing the `_source` of +the document it's expiration time won't be updated if `detect_noop` is `true`. +In 2.1 `detect_noop` defaults to `true`. diff --git a/docs/reference/migration/migrate_2_0/query_dsl.asciidoc b/docs/reference/migration/migrate_2_0/query_dsl.asciidoc index 31283e9ce33..cb9e1581a91 100644 --- a/docs/reference/migration/migrate_2_0/query_dsl.asciidoc +++ b/docs/reference/migration/migrate_2_0/query_dsl.asciidoc @@ -178,7 +178,7 @@ The parameter `percent_terms_to_match` has been removed in favor of The `limit` filter is deprecated and becomes a no-op. You can achieve similar behaviour using the <> parameter. -==== Jave plugins registering custom queries +==== Java plugins registering custom queries Java plugins that register custom queries can do so by using the `IndicesQueriesModule#addQuery(Class)` method. Other diff --git a/docs/reference/migration/migrate_2_0/removals.asciidoc b/docs/reference/migration/migrate_2_0/removals.asciidoc index f02bf3a3f7f..379565cb90e 100644 --- a/docs/reference/migration/migrate_2_0/removals.asciidoc +++ b/docs/reference/migration/migrate_2_0/removals.asciidoc @@ -40,6 +40,24 @@ can install the plugin with: ./bin/plugin install delete-by-query ------------------ +See {plugins}/plugins-delete-by-query.html for more information. + +==== Multicast Discovery is now a plugin + +Support for multicast is very patchy. Linux doesn’t allow multicast listening on localhost, +while OS/X sends multicast broadcasts across all interfaces regardless of the configured +bind address. On top of that, some networks have multicast disabled by default. + +This feature has been moved to a plugin. The default discovery mechanism now uses +unicast, with a default setup which looks for the first 5 ports on localhost. If you +still need to use multicast discovery, you can install the plugin with: + +[source,sh] +------------------ +./bin/plugin install discovery-multicast +------------------ + +See {plugins}/discovery-multicast.html for more information. ==== `_shutdown` API diff --git a/docs/reference/migration/migrate_2_1.asciidoc b/docs/reference/migration/migrate_2_1.asciidoc index 906d51afdb7..63092d9250e 100644 --- a/docs/reference/migration/migrate_2_1.asciidoc +++ b/docs/reference/migration/migrate_2_1.asciidoc @@ -24,4 +24,14 @@ GET /my_index/_search?scroll=2m Scroll requests sorted by `_doc` have been optimized to more efficiently resume from where the previous request stopped, so this will have the same performance -characteristics as the former `scan` search type. \ No newline at end of file +characteristics as the former `scan` search type. + +=== Update changes + +==== Updates now `detect_noop` by default + +We've switched the default value of the `detect_noop` option from `false` to +`true`. This means that Elasticsearch will ignore updates that don't change +source unless you explicitly set `"detect_noop": false`. `detect_noop` was +always computationally cheap compared to the expense of the update which can be +thought of as a delete operation followed by an index operation. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_warmer.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_warmer.json index d1dce8bd0e5..9039367d15f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_warmer.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_warmer.json @@ -41,7 +41,7 @@ }, "request_cache": { "type" : "boolean", - "description" : "Specify whether the request to be wamred shoyd use the request cache, defaults to index level setting" + "description" : "Specify whether the request to be warmed should use the request cache, defaults to index level setting" } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml index cdd5d7e1d0f..09179ebd5ac 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml @@ -56,7 +56,7 @@ - lte: { _ttl: 100000} - gt: { _ttl: 10000} -# duration +# seconds - do: update: @@ -66,6 +66,7 @@ body: doc: { foo: baz } upsert: { foo: bar } + detect_noop: false ttl: 20s - do: @@ -89,4 +90,3 @@ body: { foo: bar } ttl: 20s timestamp: 2013-06-23T18:14:40 -