Merge branch 'master' into pr/16598-register-filter-settings
This commit is contained in:
commit
e35032950e
|
@ -310,7 +310,7 @@ public class AllocationService extends AbstractComponent {
|
|||
}
|
||||
|
||||
// move shards that no longer can be allocated
|
||||
changed |= moveShards(allocation);
|
||||
changed |= shardsAllocators.moveShards(allocation);
|
||||
|
||||
// rebalance
|
||||
changed |= shardsAllocators.rebalance(allocation);
|
||||
|
@ -327,46 +327,6 @@ public class AllocationService extends AbstractComponent {
|
|||
}
|
||||
}
|
||||
|
||||
private boolean moveShards(RoutingAllocation allocation) {
|
||||
boolean changed = false;
|
||||
|
||||
// create a copy of the shards interleaving between nodes, and check if they can remain
|
||||
List<ShardRouting> shards = new ArrayList<>();
|
||||
int index = 0;
|
||||
boolean found = true;
|
||||
final RoutingNodes routingNodes = allocation.routingNodes();
|
||||
while (found) {
|
||||
found = false;
|
||||
for (RoutingNode routingNode : routingNodes) {
|
||||
if (index >= routingNode.size()) {
|
||||
continue;
|
||||
}
|
||||
found = true;
|
||||
shards.add(routingNode.get(index));
|
||||
}
|
||||
index++;
|
||||
}
|
||||
for (int i = 0; i < shards.size(); i++) {
|
||||
ShardRouting shardRouting = shards.get(i);
|
||||
// we can only move started shards...
|
||||
if (!shardRouting.started()) {
|
||||
continue;
|
||||
}
|
||||
final RoutingNode routingNode = routingNodes.node(shardRouting.currentNodeId());
|
||||
Decision decision = allocation.deciders().canRemain(shardRouting, routingNode, allocation);
|
||||
if (decision.type() == Decision.Type.NO) {
|
||||
logger.debug("[{}][{}] allocated on [{}], but can no longer be allocated on it, moving...", shardRouting.index(), shardRouting.id(), routingNode.node());
|
||||
boolean moved = shardsAllocators.move(shardRouting, routingNode, allocation);
|
||||
if (!moved) {
|
||||
logger.debug("[{}][{}] can't move", shardRouting.index(), shardRouting.id());
|
||||
} else {
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
private boolean electPrimariesAndUnassignedDanglingReplicas(RoutingAllocation allocation) {
|
||||
boolean changed = false;
|
||||
final RoutingNodes routingNodes = allocation.routingNodes();
|
||||
|
|
|
@ -43,6 +43,7 @@ import org.elasticsearch.common.settings.Setting.Property;
|
|||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.gateway.PriorityComparator;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
|
@ -50,6 +51,7 @@ import java.util.HashMap;
|
|||
import java.util.HashSet;
|
||||
import java.util.IdentityHashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Predicate;
|
||||
|
@ -124,9 +126,9 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
|
|||
}
|
||||
|
||||
@Override
|
||||
public boolean move(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
|
||||
public boolean moveShards(RoutingAllocation allocation) {
|
||||
final Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold);
|
||||
return balancer.move(shardRouting, node);
|
||||
return balancer.moveShards();
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -494,56 +496,93 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
|
|||
}
|
||||
|
||||
/**
|
||||
* This function executes a move operation moving the given shard from
|
||||
* the given node to the minimal eligible node with respect to the
|
||||
* weight function. Iff the shard is moved the shard will be set to
|
||||
* Move started shards that can not be allocated to a node anymore
|
||||
*
|
||||
* For each shard to be moved this function executes a move operation
|
||||
* to the minimal eligible node with respect to the
|
||||
* weight function. If a shard is moved the shard will be set to
|
||||
* {@link ShardRoutingState#RELOCATING} and a shadow instance of this
|
||||
* shard is created with an incremented version in the state
|
||||
* {@link ShardRoutingState#INITIALIZING}.
|
||||
*
|
||||
* @return <code>true</code> iff the shard has successfully been moved.
|
||||
* @return <code>true</code> if the allocation has changed, otherwise <code>false</code>
|
||||
*/
|
||||
public boolean move(ShardRouting shard, RoutingNode node ) {
|
||||
if (nodes.isEmpty() || !shard.started()) {
|
||||
/* with no nodes or a not started shard this is pointless */
|
||||
public boolean moveShards() {
|
||||
if (nodes.isEmpty()) {
|
||||
/* with no nodes this is pointless */
|
||||
return false;
|
||||
}
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Try moving shard [{}] from [{}]", shard, node);
|
||||
}
|
||||
final RoutingNodes.UnassignedShards unassigned = routingNodes.unassigned();
|
||||
boolean changed = initialize(routingNodes, unassigned);
|
||||
if (!changed) {
|
||||
final ModelNode sourceNode = nodes.get(node.nodeId());
|
||||
assert sourceNode != null;
|
||||
final NodeSorter sorter = newNodeSorter();
|
||||
sorter.reset(shard.getIndexName());
|
||||
final ModelNode[] nodes = sorter.modelNodes;
|
||||
assert sourceNode.containsShard(shard);
|
||||
/*
|
||||
* the sorter holds the minimum weight node first for the shards index.
|
||||
* We now walk through the nodes until we find a node to allocate the shard.
|
||||
* This is not guaranteed to be balanced after this operation we still try best effort to
|
||||
* allocate on the minimal eligible node.
|
||||
*/
|
||||
|
||||
for (ModelNode currentNode : nodes) {
|
||||
if (currentNode.getNodeId().equals(node.nodeId())) {
|
||||
// Create a copy of the started shards interleaving between nodes, and check if they can remain. In the presence of throttling
|
||||
// shard movements, the goal of this iteration order is to achieve a fairer movement of shards from the nodes that are
|
||||
// offloading the shards.
|
||||
List<ShardRouting> shards = new ArrayList<>();
|
||||
int index = 0;
|
||||
boolean found = true;
|
||||
while (found) {
|
||||
found = false;
|
||||
for (RoutingNode routingNode : routingNodes) {
|
||||
if (index >= routingNode.size()) {
|
||||
continue;
|
||||
}
|
||||
RoutingNode target = currentNode.getRoutingNode(routingNodes);
|
||||
Decision allocationDecision = allocation.deciders().canAllocate(shard, target, allocation);
|
||||
Decision rebalanceDecision = allocation.deciders().canRebalance(shard, allocation);
|
||||
Decision decision = new Decision.Multi().add(allocationDecision).add(rebalanceDecision);
|
||||
if (decision.type() == Type.YES) { // TODO maybe we can respect throttling here too?
|
||||
sourceNode.removeShard(shard);
|
||||
ShardRouting targetRelocatingShard = routingNodes.relocate(shard, target.nodeId(), allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
|
||||
currentNode.addShard(targetRelocatingShard, decision);
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Moved shard [{}] to node [{}]", shard, currentNode.getNodeId());
|
||||
found = true;
|
||||
ShardRouting shardRouting = routingNode.get(index);
|
||||
// we can only move started shards...
|
||||
if (shardRouting.started()) {
|
||||
shards.add(shardRouting);
|
||||
}
|
||||
}
|
||||
index++;
|
||||
}
|
||||
if (shards.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
final RoutingNodes.UnassignedShards unassigned = routingNodes.unassigned();
|
||||
boolean changed = initialize(routingNodes, unassigned);
|
||||
if (changed == false) {
|
||||
final NodeSorter sorter = newNodeSorter();
|
||||
final ModelNode[] modelNodes = sorter.modelNodes;
|
||||
for (ShardRouting shardRouting : shards) {
|
||||
final ModelNode sourceNode = nodes.get(shardRouting.currentNodeId());
|
||||
assert sourceNode != null && sourceNode.containsShard(shardRouting);
|
||||
final RoutingNode routingNode = sourceNode.getRoutingNode(routingNodes);
|
||||
Decision decision = allocation.deciders().canRemain(shardRouting, routingNode, allocation);
|
||||
if (decision.type() == Decision.Type.NO) {
|
||||
logger.debug("[{}][{}] allocated on [{}], but can no longer be allocated on it, moving...", shardRouting.index(), shardRouting.id(), routingNode.node());
|
||||
sorter.reset(shardRouting.getIndexName());
|
||||
/*
|
||||
* the sorter holds the minimum weight node first for the shards index.
|
||||
* We now walk through the nodes until we find a node to allocate the shard.
|
||||
* This is not guaranteed to be balanced after this operation we still try best effort to
|
||||
* allocate on the minimal eligible node.
|
||||
*/
|
||||
boolean moved = false;
|
||||
for (ModelNode currentNode : modelNodes) {
|
||||
if (currentNode == sourceNode) {
|
||||
continue;
|
||||
}
|
||||
RoutingNode target = currentNode.getRoutingNode(routingNodes);
|
||||
Decision allocationDecision = allocation.deciders().canAllocate(shardRouting, target, allocation);
|
||||
Decision rebalanceDecision = allocation.deciders().canRebalance(shardRouting, allocation);
|
||||
if (allocationDecision.type() == Type.YES && rebalanceDecision.type() == Type.YES) { // TODO maybe we can respect throttling here too?
|
||||
Decision sourceDecision = sourceNode.removeShard(shardRouting);
|
||||
ShardRouting targetRelocatingShard = routingNodes.relocate(shardRouting, target.nodeId(), allocation.clusterInfo().getShardSize(shardRouting, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
|
||||
// re-add (now relocating shard) to source node
|
||||
sourceNode.addShard(shardRouting, sourceDecision);
|
||||
Decision targetDecision = new Decision.Multi().add(allocationDecision).add(rebalanceDecision);
|
||||
currentNode.addShard(targetRelocatingShard, targetDecision);
|
||||
if (logger.isTraceEnabled()) {
|
||||
logger.trace("Moved shard [{}] to node [{}]", shardRouting, routingNode.node());
|
||||
}
|
||||
moved = true;
|
||||
changed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (moved == false) {
|
||||
logger.debug("[{}][{}] can't move", shardRouting.index(), shardRouting.id());
|
||||
}
|
||||
changed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,7 +19,6 @@
|
|||
|
||||
package org.elasticsearch.cluster.routing.allocation.allocator;
|
||||
|
||||
import org.elasticsearch.cluster.routing.RoutingNode;
|
||||
import org.elasticsearch.cluster.routing.ShardRouting;
|
||||
import org.elasticsearch.cluster.routing.ShardRoutingState;
|
||||
import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
|
||||
|
@ -36,22 +35,22 @@ import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
|
|||
public interface ShardsAllocator {
|
||||
|
||||
/**
|
||||
* Applies changes on started nodes based on the implemented algorithm. For example if a
|
||||
* shard has changed to {@link ShardRoutingState#STARTED} from {@link ShardRoutingState#RELOCATING}
|
||||
* Applies changes on started nodes based on the implemented algorithm. For example if a
|
||||
* shard has changed to {@link ShardRoutingState#STARTED} from {@link ShardRoutingState#RELOCATING}
|
||||
* this allocator might apply some cleanups on the node that used to hold the shard.
|
||||
* @param allocation all started {@link ShardRouting shards}
|
||||
*/
|
||||
void applyStartedShards(StartedRerouteAllocation allocation);
|
||||
|
||||
/**
|
||||
* Applies changes on failed nodes based on the implemented algorithm.
|
||||
* Applies changes on failed nodes based on the implemented algorithm.
|
||||
* @param allocation all failed {@link ShardRouting shards}
|
||||
*/
|
||||
void applyFailedShards(FailedRerouteAllocation allocation);
|
||||
|
||||
/**
|
||||
* Assign all unassigned shards to nodes
|
||||
*
|
||||
* Assign all unassigned shards to nodes
|
||||
*
|
||||
* @param allocation current node allocation
|
||||
* @return <code>true</code> if the allocation has changed, otherwise <code>false</code>
|
||||
*/
|
||||
|
@ -59,19 +58,17 @@ public interface ShardsAllocator {
|
|||
|
||||
/**
|
||||
* Rebalancing number of shards on all nodes
|
||||
*
|
||||
*
|
||||
* @param allocation current node allocation
|
||||
* @return <code>true</code> if the allocation has changed, otherwise <code>false</code>
|
||||
*/
|
||||
boolean rebalance(RoutingAllocation allocation);
|
||||
|
||||
/**
|
||||
* Moves a shard from the given node to other node.
|
||||
*
|
||||
* @param shardRouting the shard to move
|
||||
* @param node A node containing the shard
|
||||
* Move started shards that can not be allocated to a node anymore
|
||||
*
|
||||
* @param allocation current node allocation
|
||||
* @return <code>true</code> if the allocation has changed, otherwise <code>false</code>
|
||||
*/
|
||||
boolean move(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation);
|
||||
boolean moveShards(RoutingAllocation allocation);
|
||||
}
|
||||
|
|
|
@ -19,8 +19,6 @@
|
|||
|
||||
package org.elasticsearch.cluster.routing.allocation.allocator;
|
||||
|
||||
import org.elasticsearch.cluster.routing.RoutingNode;
|
||||
import org.elasticsearch.cluster.routing.ShardRouting;
|
||||
import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
|
||||
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
|
||||
import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
|
||||
|
@ -96,7 +94,7 @@ public class ShardsAllocators extends AbstractComponent implements ShardsAllocat
|
|||
}
|
||||
|
||||
@Override
|
||||
public boolean move(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
|
||||
return allocator.move(shardRouting, node, allocation);
|
||||
public boolean moveShards(RoutingAllocation allocation) {
|
||||
return allocator.moveShards(allocation);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -98,7 +98,6 @@ class DocumentParser implements Closeable {
|
|||
}
|
||||
|
||||
reverseOrder(context);
|
||||
applyDocBoost(context);
|
||||
|
||||
ParsedDocument doc = parsedDocument(source, context, update(context, mapping));
|
||||
// reset the context to free up memory
|
||||
|
@ -186,24 +185,6 @@ class DocumentParser implements Closeable {
|
|||
}
|
||||
}
|
||||
|
||||
private static void applyDocBoost(ParseContext.InternalParseContext context) {
|
||||
// apply doc boost
|
||||
if (context.docBoost() != 1.0f) {
|
||||
Set<String> encounteredFields = new HashSet<>();
|
||||
for (ParseContext.Document doc : context.docs()) {
|
||||
encounteredFields.clear();
|
||||
for (IndexableField field : doc) {
|
||||
if (field.fieldType().indexOptions() != IndexOptions.NONE && !field.fieldType().omitNorms()) {
|
||||
if (!encounteredFields.contains(field.name())) {
|
||||
((Field) field).setBoost(context.docBoost() * field.boost());
|
||||
encounteredFields.add(field.name());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static ParsedDocument parsedDocument(SourceToParse source, ParseContext.InternalParseContext context, Mapping update) {
|
||||
return new ParsedDocument(
|
||||
context.uid(),
|
||||
|
|
|
@ -303,7 +303,8 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
|
|||
for (Field field : fields) {
|
||||
if (!customBoost()
|
||||
// don't set boosts eg. on dv fields
|
||||
&& field.fieldType().indexOptions() != IndexOptions.NONE) {
|
||||
&& field.fieldType().indexOptions() != IndexOptions.NONE
|
||||
&& Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
|
||||
field.setBoost(fieldType().boost());
|
||||
}
|
||||
context.doc().add(field);
|
||||
|
|
|
@ -32,7 +32,9 @@ import org.apache.lucene.search.Query;
|
|||
import org.apache.lucene.search.RegexpQuery;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.search.TermRangeQuery;
|
||||
import org.apache.lucene.search.BoostQuery;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.action.fieldstats.FieldStats;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.Strings;
|
||||
|
@ -398,7 +400,12 @@ public abstract class MappedFieldType extends FieldType {
|
|||
}
|
||||
|
||||
public Query termQuery(Object value, @Nullable QueryShardContext context) {
|
||||
return new TermQuery(createTerm(value));
|
||||
TermQuery query = new TermQuery(createTerm(value));
|
||||
if (boost == 1f ||
|
||||
(context != null && context.indexVersionCreated().before(Version.V_5_0_0))) {
|
||||
return query;
|
||||
}
|
||||
return new BoostQuery(query, boost);
|
||||
}
|
||||
|
||||
public Query termsQuery(List values, @Nullable QueryShardContext context) {
|
||||
|
|
|
@ -321,16 +321,6 @@ public abstract class ParseContext {
|
|||
return in.externalValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
public float docBoost() {
|
||||
return in.docBoost();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void docBoost(float docBoost) {
|
||||
in.docBoost(docBoost);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StringBuilder stringBuilder() {
|
||||
return in.stringBuilder();
|
||||
|
@ -375,8 +365,6 @@ public abstract class ParseContext {
|
|||
|
||||
private AllEntries allEntries = new AllEntries();
|
||||
|
||||
private float docBoost = 1.0f;
|
||||
|
||||
private Mapper dynamicMappingsUpdate = null;
|
||||
|
||||
public InternalParseContext(@Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) {
|
||||
|
@ -402,7 +390,6 @@ public abstract class ParseContext {
|
|||
this.source = source == null ? null : sourceToParse.source();
|
||||
this.path.reset();
|
||||
this.allEntries = new AllEntries();
|
||||
this.docBoost = 1.0f;
|
||||
this.dynamicMappingsUpdate = null;
|
||||
}
|
||||
|
||||
|
@ -534,16 +521,6 @@ public abstract class ParseContext {
|
|||
return this.allEntries;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float docBoost() {
|
||||
return this.docBoost;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void docBoost(float docBoost) {
|
||||
this.docBoost = docBoost;
|
||||
}
|
||||
|
||||
/**
|
||||
* A string builder that can be used to construct complex names for example.
|
||||
* Its better to reuse the.
|
||||
|
@ -759,10 +736,6 @@ public abstract class ParseContext {
|
|||
return clazz.cast(externalValue());
|
||||
}
|
||||
|
||||
public abstract float docBoost();
|
||||
|
||||
public abstract void docBoost(float docBoost);
|
||||
|
||||
/**
|
||||
* A string builder that can be used to construct complex names for example.
|
||||
* Its better to reuse the.
|
||||
|
|
|
@ -285,7 +285,9 @@ public class ByteFieldMapper extends NumberFieldMapper {
|
|||
}
|
||||
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
|
||||
CustomByteNumericField field = new CustomByteNumericField(value, fieldType());
|
||||
field.setBoost(boost);
|
||||
if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
|
||||
field.setBoost(boost);
|
||||
}
|
||||
fields.add(field);
|
||||
}
|
||||
if (fieldType().hasDocValues()) {
|
||||
|
|
|
@ -513,7 +513,9 @@ public class DateFieldMapper extends NumberFieldMapper {
|
|||
if (value != null) {
|
||||
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
|
||||
CustomLongNumericField field = new CustomLongNumericField(value, fieldType());
|
||||
field.setBoost(boost);
|
||||
if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
|
||||
field.setBoost(boost);
|
||||
}
|
||||
fields.add(field);
|
||||
}
|
||||
if (fieldType().hasDocValues()) {
|
||||
|
|
|
@ -278,7 +278,9 @@ public class DoubleFieldMapper extends NumberFieldMapper {
|
|||
|
||||
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
|
||||
CustomDoubleNumericField field = new CustomDoubleNumericField(value, fieldType());
|
||||
field.setBoost(boost);
|
||||
if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
|
||||
field.setBoost(boost);
|
||||
}
|
||||
fields.add(field);
|
||||
}
|
||||
if (fieldType().hasDocValues()) {
|
||||
|
|
|
@ -290,7 +290,9 @@ public class FloatFieldMapper extends NumberFieldMapper {
|
|||
|
||||
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
|
||||
CustomFloatNumericField field = new CustomFloatNumericField(value, fieldType());
|
||||
field.setBoost(boost);
|
||||
if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
|
||||
field.setBoost(boost);
|
||||
}
|
||||
fields.add(field);
|
||||
}
|
||||
if (fieldType().hasDocValues()) {
|
||||
|
|
|
@ -298,7 +298,9 @@ public class IntegerFieldMapper extends NumberFieldMapper {
|
|||
protected void addIntegerFields(ParseContext context, List<Field> fields, int value, float boost) {
|
||||
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
|
||||
CustomIntegerNumericField field = new CustomIntegerNumericField(value, fieldType());
|
||||
field.setBoost(boost);
|
||||
if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
|
||||
field.setBoost(boost);
|
||||
}
|
||||
fields.add(field);
|
||||
}
|
||||
if (fieldType().hasDocValues()) {
|
||||
|
|
|
@ -282,7 +282,9 @@ public class LongFieldMapper extends NumberFieldMapper {
|
|||
}
|
||||
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
|
||||
CustomLongNumericField field = new CustomLongNumericField(value, fieldType());
|
||||
field.setBoost(boost);
|
||||
if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
|
||||
field.setBoost(boost);
|
||||
}
|
||||
fields.add(field);
|
||||
}
|
||||
if (fieldType().hasDocValues()) {
|
||||
|
|
|
@ -290,7 +290,9 @@ public class ShortFieldMapper extends NumberFieldMapper {
|
|||
}
|
||||
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
|
||||
CustomShortNumericField field = new CustomShortNumericField(value, fieldType());
|
||||
field.setBoost(boost);
|
||||
if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
|
||||
field.setBoost(boost);
|
||||
}
|
||||
fields.add(field);
|
||||
}
|
||||
if (fieldType().hasDocValues()) {
|
||||
|
|
|
@ -317,7 +317,9 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
|
|||
|
||||
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
|
||||
Field field = new Field(fieldType().name(), valueAndBoost.value(), fieldType());
|
||||
field.setBoost(valueAndBoost.boost());
|
||||
if (valueAndBoost.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
|
||||
field.setBoost(valueAndBoost.boost());
|
||||
}
|
||||
fields.add(field);
|
||||
}
|
||||
if (fieldType().hasDocValues()) {
|
||||
|
|
|
@ -30,6 +30,7 @@ import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
|
|||
import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
|
||||
import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
|
||||
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.Explicit;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.geo.GeoUtils;
|
||||
|
@ -452,7 +453,8 @@ public class GeoShapeFieldMapper extends FieldMapper {
|
|||
return null;
|
||||
}
|
||||
for (Field field : fields) {
|
||||
if (!customBoost()) {
|
||||
if (!customBoost() &&
|
||||
fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
|
||||
field.setBoost(fieldType().boost());
|
||||
}
|
||||
context.doc().add(field);
|
||||
|
|
|
@ -27,6 +27,7 @@ import org.apache.lucene.search.Query;
|
|||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.BytesRefBuilder;
|
||||
import org.apache.lucene.util.NumericUtils;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.Explicit;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.Numbers;
|
||||
|
@ -305,7 +306,9 @@ public class IpFieldMapper extends NumberFieldMapper {
|
|||
final long value = ipToLong(ipAsString);
|
||||
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
|
||||
CustomLongNumericField field = new CustomLongNumericField(value, fieldType());
|
||||
field.setBoost(fieldType().boost());
|
||||
if (fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) {
|
||||
field.setBoost(fieldType().boost());
|
||||
}
|
||||
fields.add(field);
|
||||
}
|
||||
if (fieldType().hasDocValues()) {
|
||||
|
|
|
@ -333,9 +333,18 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
|
|||
// callback that an exception happened, but on a different thread since we don't
|
||||
// want handlers to worry about stack overflows
|
||||
final SendRequestTransportException sendRequestException = new SendRequestTransportException(node, action, e);
|
||||
threadPool.executor(ThreadPool.Names.GENERIC).execute(new Runnable() {
|
||||
threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
public void onRejection(Throwable t) {
|
||||
// if we get rejected during node shutdown we don't wanna bubble it up
|
||||
logger.debug("failed to notify response handler on rejection", t);
|
||||
}
|
||||
@Override
|
||||
public void onFailure(Throwable t) {
|
||||
logger.warn("failed to notify response handler on exception", t);
|
||||
}
|
||||
@Override
|
||||
protected void doRun() throws Exception {
|
||||
holderToNotify.handler().handleException(sendRequestException);
|
||||
}
|
||||
});
|
||||
|
|
|
@ -477,7 +477,7 @@ public class IndexAliasesIT extends ESIntegTestCase {
|
|||
});
|
||||
}
|
||||
executor.shutdown();
|
||||
boolean done = executor.awaitTermination(10, TimeUnit.SECONDS);
|
||||
boolean done = executor.awaitTermination(20, TimeUnit.SECONDS);
|
||||
assertThat(done, equalTo(true));
|
||||
if (!done) {
|
||||
executor.shutdownNow();
|
||||
|
|
|
@ -61,7 +61,7 @@ public class ClusterModuleTests extends ModuleTestCase {
|
|||
return false;
|
||||
}
|
||||
@Override
|
||||
public boolean move(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
|
||||
public boolean moveShards(RoutingAllocation allocation) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -320,7 +320,7 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
|
|||
}
|
||||
|
||||
@Override
|
||||
public boolean move(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
|
||||
public boolean moveShards(RoutingAllocation allocation) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
|
@ -19,13 +19,18 @@
|
|||
|
||||
package org.elasticsearch.index.mapper.boost;
|
||||
|
||||
import org.apache.lucene.search.BoostQuery;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
||||
import org.elasticsearch.common.compress.CompressedXContent;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.index.IndexService;
|
||||
import org.elasticsearch.index.mapper.DocumentFieldMappers;
|
||||
import org.elasticsearch.index.mapper.DocumentMapper;
|
||||
import org.elasticsearch.index.mapper.ParsedDocument;
|
||||
import org.elasticsearch.index.query.QueryShardContext;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.test.ESSingleNodeTestCase;
|
||||
import org.elasticsearch.test.InternalSettingsPlugin;
|
||||
|
@ -33,6 +38,7 @@ import org.elasticsearch.test.InternalSettingsPlugin;
|
|||
import java.util.Collection;
|
||||
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
|
||||
public class CustomBoostMappingTests extends ESSingleNodeTestCase {
|
||||
|
||||
|
@ -77,4 +83,87 @@ public class CustomBoostMappingTests extends ESSingleNodeTestCase {
|
|||
assertThat(doc.rootDoc().getField("f_field").boost(), equalTo(8.0f));
|
||||
assertThat(doc.rootDoc().getField("date_field").boost(), equalTo(9.0f));
|
||||
}
|
||||
|
||||
public void testBackCompatFieldMappingBoostValues() throws Exception {
|
||||
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
|
||||
.startObject("s_field").field("type", "keyword").field("boost", 2.0f).endObject()
|
||||
.startObject("l_field").field("type", "long").field("boost", 3.0f).startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("i_field").field("type", "integer").field("boost", 4.0f).startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("sh_field").field("type", "short").field("boost", 5.0f).startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("b_field").field("type", "byte").field("boost", 6.0f).startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("d_field").field("type", "double").field("boost", 7.0f).startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("f_field").field("type", "float").field("boost", 8.0f).startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("date_field").field("type", "date").field("boost", 9.0f).startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.endObject().endObject().endObject().string();
|
||||
|
||||
{
|
||||
IndexService indexService = createIndex("test", BW_SETTINGS);
|
||||
QueryShardContext context = indexService.newQueryShardContext();
|
||||
DocumentMapper mapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
|
||||
DocumentFieldMappers fieldMappers = mapper.mappers();
|
||||
assertThat(fieldMappers.getMapper("s_field").fieldType().termQuery("0", context), instanceOf(TermQuery.class));
|
||||
assertThat(fieldMappers.getMapper("l_field").fieldType().termQuery("0", context), instanceOf(TermQuery.class));
|
||||
assertThat(fieldMappers.getMapper("i_field").fieldType().termQuery("0", context), instanceOf(TermQuery.class));
|
||||
assertThat(fieldMappers.getMapper("sh_field").fieldType().termQuery("0", context), instanceOf(TermQuery.class));
|
||||
assertThat(fieldMappers.getMapper("b_field").fieldType().termQuery("0", context), instanceOf(TermQuery.class));
|
||||
assertThat(fieldMappers.getMapper("d_field").fieldType().termQuery("0", context), instanceOf(TermQuery.class));
|
||||
assertThat(fieldMappers.getMapper("f_field").fieldType().termQuery("0", context), instanceOf(TermQuery.class));
|
||||
assertThat(fieldMappers.getMapper("date_field").fieldType().termQuery("0", context), instanceOf(TermQuery.class));
|
||||
|
||||
ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder().startObject()
|
||||
.field("s_field", "s_value")
|
||||
.field("l_field", 1L)
|
||||
.field("i_field", 1)
|
||||
.field("sh_field", 1)
|
||||
.field("b_field", 1)
|
||||
.field("d_field", 1)
|
||||
.field("f_field", 1)
|
||||
.field("date_field", "20100101")
|
||||
.endObject().bytes());
|
||||
|
||||
assertThat(doc.rootDoc().getField("s_field").boost(), equalTo(2.0f));
|
||||
assertThat(doc.rootDoc().getField("l_field").boost(), equalTo(3.0f));
|
||||
assertThat(doc.rootDoc().getField("i_field").boost(), equalTo(4.0f));
|
||||
assertThat(doc.rootDoc().getField("sh_field").boost(), equalTo(5.0f));
|
||||
assertThat(doc.rootDoc().getField("b_field").boost(), equalTo(6.0f));
|
||||
assertThat(doc.rootDoc().getField("d_field").boost(), equalTo(7.0f));
|
||||
assertThat(doc.rootDoc().getField("f_field").boost(), equalTo(8.0f));
|
||||
assertThat(doc.rootDoc().getField("date_field").boost(), equalTo(9.0f));
|
||||
}
|
||||
|
||||
{
|
||||
IndexService indexService = createIndex("text");
|
||||
QueryShardContext context = indexService.newQueryShardContext();
|
||||
DocumentMapper mapper = indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
|
||||
DocumentFieldMappers fieldMappers = mapper.mappers();
|
||||
assertThat(fieldMappers.getMapper("s_field").fieldType().termQuery("0", context), instanceOf(BoostQuery.class));
|
||||
assertThat(fieldMappers.getMapper("l_field").fieldType().termQuery("0", context), instanceOf(BoostQuery.class));
|
||||
assertThat(fieldMappers.getMapper("i_field").fieldType().termQuery("0", context), instanceOf(BoostQuery.class));
|
||||
assertThat(fieldMappers.getMapper("sh_field").fieldType().termQuery("0", context), instanceOf(BoostQuery.class));
|
||||
assertThat(fieldMappers.getMapper("b_field").fieldType().termQuery("0", context), instanceOf(BoostQuery.class));
|
||||
assertThat(fieldMappers.getMapper("d_field").fieldType().termQuery("0", context), instanceOf(BoostQuery.class));
|
||||
assertThat(fieldMappers.getMapper("f_field").fieldType().termQuery("0", context), instanceOf(BoostQuery.class));
|
||||
assertThat(fieldMappers.getMapper("date_field").fieldType().termQuery("0", context), instanceOf(BoostQuery.class));
|
||||
|
||||
ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder().startObject()
|
||||
.field("s_field", "s_value")
|
||||
.field("l_field", 1L)
|
||||
.field("i_field", 1)
|
||||
.field("sh_field", 1)
|
||||
.field("b_field", 1)
|
||||
.field("d_field", 1)
|
||||
.field("f_field", 1)
|
||||
.field("date_field", "20100101")
|
||||
.endObject().bytes());
|
||||
|
||||
assertThat(doc.rootDoc().getField("s_field").boost(), equalTo(1f));
|
||||
assertThat(doc.rootDoc().getField("l_field").boost(), equalTo(1f));
|
||||
assertThat(doc.rootDoc().getField("i_field").boost(), equalTo(1f));
|
||||
assertThat(doc.rootDoc().getField("sh_field").boost(), equalTo(1f));
|
||||
assertThat(doc.rootDoc().getField("b_field").boost(), equalTo(1f));
|
||||
assertThat(doc.rootDoc().getField("d_field").boost(), equalTo(1f));
|
||||
assertThat(doc.rootDoc().getField("f_field").boost(), equalTo(1f));
|
||||
assertThat(doc.rootDoc().getField("date_field").boost(), equalTo(1f));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -36,6 +36,7 @@ import org.elasticsearch.test.InternalSettingsPlugin;
|
|||
import java.util.Collection;
|
||||
|
||||
import static org.hamcrest.Matchers.closeTo;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
|
||||
/**
|
||||
*/
|
||||
|
@ -98,6 +99,97 @@ public class FieldLevelBoostTests extends ESSingleNodeTestCase {
|
|||
assertThat((double) f.boost(), closeTo(9.0, 0.001));
|
||||
}
|
||||
|
||||
public void testBackCompatFieldLevelMappingBoost() throws Exception {
|
||||
String mapping = XContentFactory.jsonBuilder().startObject().startObject("person").startObject("properties")
|
||||
.startObject("str_field").field("type", "keyword").field("boost", "2.0").endObject()
|
||||
.startObject("int_field").field("type", "integer").field("boost", "3.0").startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("byte_field").field("type", "byte").field("boost", "4.0").startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("date_field").field("type", "date").field("boost", "5.0").startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("double_field").field("type", "double").field("boost", "6.0").startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("float_field").field("type", "float").field("boost", "7.0").startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("long_field").field("type", "long").field("boost", "8.0").startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.startObject("short_field").field("type", "short").field("boost", "9.0").startObject("norms").field("enabled", true).endObject().endObject()
|
||||
.string();
|
||||
|
||||
{
|
||||
DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("person", new CompressedXContent(mapping));
|
||||
BytesReference json = XContentFactory.jsonBuilder().startObject()
|
||||
.field("str_field", "some name")
|
||||
.field("int_field", 10)
|
||||
.field("byte_field", 20)
|
||||
.field("date_field", "2012-01-10")
|
||||
.field("double_field", 30.0)
|
||||
.field("float_field", 40.0)
|
||||
.field("long_field", 50)
|
||||
.field("short_field", 60)
|
||||
.bytes();
|
||||
Document doc = docMapper.parse("test", "person", "1", json).rootDoc();
|
||||
|
||||
IndexableField f = doc.getField("str_field");
|
||||
assertThat((double) f.boost(), closeTo(2.0, 0.001));
|
||||
|
||||
f = doc.getField("int_field");
|
||||
assertThat((double) f.boost(), closeTo(3.0, 0.001));
|
||||
|
||||
f = doc.getField("byte_field");
|
||||
assertThat((double) f.boost(), closeTo(4.0, 0.001));
|
||||
|
||||
f = doc.getField("date_field");
|
||||
assertThat((double) f.boost(), closeTo(5.0, 0.001));
|
||||
|
||||
f = doc.getField("double_field");
|
||||
assertThat((double) f.boost(), closeTo(6.0, 0.001));
|
||||
|
||||
f = doc.getField("float_field");
|
||||
assertThat((double) f.boost(), closeTo(7.0, 0.001));
|
||||
|
||||
f = doc.getField("long_field");
|
||||
assertThat((double) f.boost(), closeTo(8.0, 0.001));
|
||||
|
||||
f = doc.getField("short_field");
|
||||
assertThat((double) f.boost(), closeTo(9.0, 0.001));
|
||||
}
|
||||
|
||||
{
|
||||
DocumentMapper docMapper = createIndex("test2").mapperService().documentMapperParser().parse("person", new CompressedXContent(mapping));
|
||||
BytesReference json = XContentFactory.jsonBuilder().startObject()
|
||||
.field("str_field", "some name")
|
||||
.field("int_field", 10)
|
||||
.field("byte_field", 20)
|
||||
.field("date_field", "2012-01-10")
|
||||
.field("double_field", 30.0)
|
||||
.field("float_field", 40.0)
|
||||
.field("long_field", 50)
|
||||
.field("short_field", 60)
|
||||
.bytes();
|
||||
Document doc = docMapper.parse("test", "person", "1", json).rootDoc();
|
||||
|
||||
IndexableField f = doc.getField("str_field");
|
||||
assertThat(f.boost(), equalTo(1f));
|
||||
|
||||
f = doc.getField("int_field");
|
||||
assertThat(f.boost(), equalTo(1f));
|
||||
|
||||
f = doc.getField("byte_field");
|
||||
assertThat(f.boost(), equalTo(1f));
|
||||
|
||||
f = doc.getField("date_field");
|
||||
assertThat(f.boost(), equalTo(1f));
|
||||
|
||||
f = doc.getField("double_field");
|
||||
assertThat(f.boost(), equalTo(1f));
|
||||
|
||||
f = doc.getField("float_field");
|
||||
assertThat(f.boost(), equalTo(1f));
|
||||
|
||||
f = doc.getField("long_field");
|
||||
assertThat(f.boost(), equalTo(1f));
|
||||
|
||||
f = doc.getField("short_field");
|
||||
assertThat(f.boost(), equalTo(1f));
|
||||
}
|
||||
}
|
||||
|
||||
public void testBackCompatInvalidFieldLevelBoost() throws Exception {
|
||||
String mapping = XContentFactory.jsonBuilder().startObject().startObject("person").startObject("properties")
|
||||
.startObject("str_field").field("type", "string").endObject()
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
[[ingest-attachment]]
|
||||
=== Ingest Attachment Processor Plugin
|
||||
|
||||
The ingest attachment plugin lets Elasticsearch extract file attachments in common formats (such as PPT, XLS, PDF)
|
||||
The ingest attachment plugin lets Elasticsearch extract file attachments in common formats (such as PPT, XLS, and PDF) by
|
||||
using the Apache text extraction library http://lucene.apache.org/tika/[Tika].
|
||||
|
||||
It can be used as replacement for the mapper attachment plugin.
|
||||
You can use the ingest attachment plugin as a replacement for the mapper attachment plugin.
|
||||
|
||||
The source field must be a base64 encoded binary.
|
||||
|
||||
|
@ -16,7 +16,7 @@ The source field must be a base64 encoded binary.
|
|||
| `source_field` | yes | - | The field to get the base64 encoded field from
|
||||
| `target_field` | no | attachment | The field that will hold the attachment information
|
||||
| `indexed_chars` | no | 100000 | The number of chars being used for extraction to prevent huge fields. Use `-1` for no limit.
|
||||
| `fields` | no | all | Properties to select to be stored, can be `content`, `title`, `name`, `author`, `keywords`, `date`, `content_type`, `content_length`, `language`
|
||||
| `fields` | no | all | Properties to select to be stored. Can be `content`, `title`, `name`, `author`, `keywords`, `date`, `content_type`, `content_length`, `language`
|
||||
|======
|
||||
|
||||
[source,js]
|
||||
|
|
|
@ -7,7 +7,7 @@ This processor adds this information by default under the `geoip` field.
|
|||
The ingest-geoip plugin ships by default with the GeoLite2 City and GeoLite2 Country geoip2 databases from Maxmind made available
|
||||
under the CCA-ShareAlike 3.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/
|
||||
|
||||
The GeoIP processor can run with other geoip2 databases from Maxmind. The files must be copied into the geoip config directory
|
||||
The GeoIP processor can run with other geoip2 databases from Maxmind. The files must be copied into the geoip config directory,
|
||||
and the `database_file` option should be used to specify the filename of the custom database. The geoip config directory
|
||||
is located at `$ES_HOME/config/ingest/geoip` and holds the shipped databases too.
|
||||
|
||||
|
@ -24,13 +24,13 @@ is located at `$ES_HOME/config/ingest/geoip` and holds the shipped databases too
|
|||
|
||||
*Depends on what is available in `database_field`:
|
||||
|
||||
* If the GeoLite2 City database is used then the following fields may be added under the `target_field`: `ip`,
|
||||
* If the GeoLite2 City database is used, then the following fields may be added under the `target_field`: `ip`,
|
||||
`country_iso_code`, `country_name`, `continent_name`, `region_name`, `city_name`, `timezone`, `latitude`, `longitude`
|
||||
and `location`. The fields actually added depend on what has been found and which fields were configured in `fields`.
|
||||
* If the GeoLite2 Country database is used then the following fields may be added under the `target_field`: `ip`,
|
||||
`country_iso_code`, `country_name` and `continent_name`.The fields actually added depend on what has been found and which fields were configured in `fields`.
|
||||
* If the GeoLite2 Country database is used, then the following fields may be added under the `target_field`: `ip`,
|
||||
`country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which fields were configured in `fields`.
|
||||
|
||||
An example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:
|
||||
Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -46,7 +46,7 @@ An example that uses the default city database and adds the geographical informa
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
An example that uses the default country database and add the geographical information to the `geo` field based on the `ip` field`:
|
||||
Here is an example that uses the default country database and adds the geographical information to the `geo` field based on the `ip` field`:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
|
|
@ -268,7 +268,7 @@ POST /_reindex
|
|||
"company": "cat"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"index": {
|
||||
"index": "dest",
|
||||
"routing": "=cat"
|
||||
|
@ -277,6 +277,23 @@ POST /_reindex
|
|||
--------------------------------------------------
|
||||
// AUTOSENSE
|
||||
|
||||
Reindex can also use the <<ingest>> feature by specifying a
|
||||
`pipeline` like this:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
POST /_reindex
|
||||
{
|
||||
"source": {
|
||||
"index": "source"
|
||||
},
|
||||
"index": {
|
||||
"index": "dest",
|
||||
"pipeline": "some_ingest_pipeline"
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
// AUTOSENSE
|
||||
|
||||
[float]
|
||||
=== URL Parameters
|
||||
|
|
|
@ -138,6 +138,15 @@ POST /twitter/_update_by_query?scroll_size=1000
|
|||
--------------------------------------------------
|
||||
// AUTOSENSE
|
||||
|
||||
`_update_by_query` can also use the <<ingest>> feature by
|
||||
specifying a `pipeline` like this:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
POST /twitter/_update_by_query?pipeline=some_ingest_pipeline
|
||||
--------------------------------------------------
|
||||
// AUTOSENSE
|
||||
|
||||
[float]
|
||||
=== URL Parameters
|
||||
|
||||
|
|
|
@ -3,22 +3,25 @@
|
|||
|
||||
[partintro]
|
||||
--
|
||||
Ingest node can be used to pre-process documents before the actual indexing takes place.
|
||||
You can use ingest node to pre-process documents before the actual indexing takes place.
|
||||
This pre-processing happens by an ingest node that intercepts bulk and index requests, applies the
|
||||
transformations and then passes the documents back to the index or bulk APIs.
|
||||
transformations, and then passes the documents back to the index or bulk APIs.
|
||||
|
||||
Ingest node is enabled by default. In order to disable ingest the following
|
||||
setting should be configured in the elasticsearch.yml file:
|
||||
You can enable ingest on any node or even have dedicated ingest nodes. Ingest is enabled by default
|
||||
on all nodes. To disable ingest on a node, configure the following setting in the `elasticsearch.yml` file:
|
||||
|
||||
[source,yaml]
|
||||
--------------------------------------------------
|
||||
node.ingest: false
|
||||
--------------------------------------------------
|
||||
|
||||
It is possible to enable ingest on any node or have dedicated ingest nodes.
|
||||
To pre-process documents before indexing, you <<pipe-line,define a pipeline>> that specifies
|
||||
a series of <<ingest-processors,processors>>. Each processor transforms the document in some way.
|
||||
For example, you may have a pipeline that consists of one processor that removes a field from
|
||||
the document followed by another processor that renames a field.
|
||||
|
||||
In order to pre-process document before indexing the `pipeline` parameter should be used
|
||||
on an index or bulk request to tell Ingest what pipeline is going to be used.
|
||||
To use a pipeline, you simply specify the `pipeline` parameter on an index or bulk request to
|
||||
tell the ingest node which pipeline to use. For example:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -29,6 +32,8 @@ PUT /my-index/my-type/my-id?pipeline=my_pipeline_id
|
|||
--------------------------------------------------
|
||||
// AUTOSENSE
|
||||
|
||||
See <<ingest-apis,Ingest APIs>> for more information about creating, adding, and deleting pipelines.
|
||||
|
||||
--
|
||||
|
||||
include::ingest/ingest-node.asciidoc[]
|
|
@ -1,8 +1,10 @@
|
|||
[[pipe-line]]
|
||||
== Pipeline Definition
|
||||
|
||||
A pipeline is a definition of a series of processors that are to be
|
||||
executed in the same sequential order as they are declared.
|
||||
A pipeline is a definition of a series of <<ingest-processors, processors>> that are to be executed
|
||||
in the same order as they are declared. A pipeline consists of two main fields: a `description`
|
||||
and a list of `processors`:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
{
|
||||
|
@ -11,17 +13,26 @@ executed in the same sequential order as they are declared.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
The `description` is a special field to store a helpful description of
|
||||
what the pipeline attempts to achieve.
|
||||
The `description` is a special field to store a helpful description of
|
||||
what the pipeline does.
|
||||
|
||||
The `processors` parameter defines a list of processors to be executed in
|
||||
The `processors` parameter defines a list of processors to be executed in
|
||||
order.
|
||||
|
||||
[[ingest-apis]]
|
||||
== Ingest APIs
|
||||
|
||||
=== Put pipeline API
|
||||
The following ingest APIs are available for managing pipelines:
|
||||
|
||||
The put pipeline api adds pipelines and updates existing pipelines in the cluster.
|
||||
* <<put-pipeline-api>> to add or update a pipeline
|
||||
* <<get-pipeline-api>> to return a specific pipeline
|
||||
* <<delete-pipeline-api>> to delete a pipeline
|
||||
* <<simulate-pipeline-api>> to simulate a call to a pipeline
|
||||
|
||||
[[put-pipeline-api]]
|
||||
=== Put Pipeline API
|
||||
|
||||
The put pipeline API adds pipelines and updates existing pipelines in the cluster.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -40,12 +51,13 @@ PUT _ingest/pipeline/my-pipeline-id
|
|||
--------------------------------------------------
|
||||
// AUTOSENSE
|
||||
|
||||
NOTE: The put pipeline api also instructs all ingest nodes to reload their in-memory representation of pipelines, so that
|
||||
pipeline changes take immediately in effect.
|
||||
NOTE: The put pipeline API also instructs all ingest nodes to reload their in-memory representation of pipelines, so that
|
||||
pipeline changes take effect immediately.
|
||||
|
||||
=== Get pipeline API
|
||||
[[get-pipeline-api]]
|
||||
=== Get Pipeline API
|
||||
|
||||
The get pipeline api returns pipelines based on id. This api always returns a local reference of the pipeline.
|
||||
The get pipeline API returns pipelines based on ID. This API always returns a local reference of the pipeline.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -75,13 +87,14 @@ Example response:
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
For each returned pipeline the source and the version is returned.
|
||||
The version is useful for knowing what version of the pipeline the node has.
|
||||
Multiple ids can be provided at the same time. Also wildcards are supported.
|
||||
For each returned pipeline, the source and the version are returned.
|
||||
The version is useful for knowing which version of the pipeline the node has.
|
||||
You can specify multiple IDs to return more than one pipeline. Wildcards are also supported.
|
||||
|
||||
=== Delete pipeline API
|
||||
[[delete-pipeline-api]]
|
||||
=== Delete Pipeline API
|
||||
|
||||
The delete pipeline api deletes pipelines by id.
|
||||
The delete pipeline API deletes pipelines by ID.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -89,16 +102,18 @@ DELETE _ingest/pipeline/my-pipeline-id
|
|||
--------------------------------------------------
|
||||
// AUTOSENSE
|
||||
|
||||
=== Simulate pipeline API
|
||||
[[simulate-pipeline-api]]
|
||||
=== Simulate Pipeline API
|
||||
|
||||
The simulate pipeline api executes a specific pipeline against
|
||||
The simulate pipeline API executes a specific pipeline against
|
||||
the set of documents provided in the body of the request.
|
||||
|
||||
A simulate request may call upon an existing pipeline to be executed
|
||||
You can either specify an existing pipeline to execute
|
||||
against the provided documents, or supply a pipeline definition in
|
||||
the body of the request.
|
||||
|
||||
Here is the structure of a simulate request with a provided pipeline:
|
||||
Here is the structure of a simulate request with a pipeline definition provided
|
||||
in the body of the request:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -115,7 +130,7 @@ POST _ingest/pipeline/_simulate
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
Here is the structure of a simulate request against a pre-existing pipeline:
|
||||
Here is the structure of a simulate request against an existing pipeline:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -130,7 +145,8 @@ POST _ingest/pipeline/my-pipeline-id/_simulate
|
|||
--------------------------------------------------
|
||||
|
||||
|
||||
Here is an example simulate request with a provided pipeline and its response:
|
||||
Here is an example of a simulate request with a pipeline defined in the request
|
||||
and its response:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -170,7 +186,7 @@ POST _ingest/pipeline/_simulate
|
|||
--------------------------------------------------
|
||||
// AUTOSENSE
|
||||
|
||||
response:
|
||||
Response:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -216,13 +232,14 @@ response:
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
It is often useful to see how each processor affects the ingest document
|
||||
as it is passed through the pipeline. To see the intermediate results of
|
||||
each processor in the simulate request, a `verbose` parameter may be added
|
||||
to the request
|
||||
|
||||
Here is an example verbose request and its response:
|
||||
[[ingest-verbose-param]]
|
||||
==== Viewing Verbose Results
|
||||
You can use the simulate pipeline API to see how each processor affects the ingest document
|
||||
as it passes through the pipeline. To see the intermediate results of
|
||||
each processor in the simulate request, you can add the `verbose` parameter
|
||||
to the request.
|
||||
|
||||
Here is an example of a verbose request and its response:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -268,7 +285,7 @@ POST _ingest/pipeline/_simulate?verbose
|
|||
--------------------------------------------------
|
||||
// AUTOSENSE
|
||||
|
||||
response:
|
||||
Response:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -364,12 +381,16 @@ response:
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
== Accessing data in pipelines
|
||||
[[accessing-data-in-pipelines]]
|
||||
== Accessing Data in Pipelines
|
||||
|
||||
Processors in pipelines have read and write access to documents that pass through the pipeline.
|
||||
The fields in the source of a document and its metadata fields are accessible.
|
||||
The processors in a pipeline have read and write access to documents that pass through the pipeline.
|
||||
The processors can access fields in the source of a document and the document's metadata fields.
|
||||
|
||||
Accessing a field in the source is straightforward and one can refer to fields by
|
||||
[float]
|
||||
[[accessing-source-fields]]
|
||||
=== Accessing Fields in the Source
|
||||
Accessing a field in the source is straightforward. You simply refer to fields by
|
||||
their name. For example:
|
||||
|
||||
[source,js]
|
||||
|
@ -382,7 +403,7 @@ their name. For example:
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
On top of this fields from the source are always accessible via the `_source` prefix:
|
||||
On top of this, fields from the source are always accessible via the `_source` prefix:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -394,11 +415,14 @@ On top of this fields from the source are always accessible via the `_source` pr
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
Metadata fields can also be accessed in the same way as fields from the source. This
|
||||
[float]
|
||||
[[accessing-metadata-fields]]
|
||||
=== Accessing Metadata Fields
|
||||
You can access metadata fields in the same way that you access fields in the source. This
|
||||
is possible because Elasticsearch doesn't allow fields in the source that have the
|
||||
same name as metadata fields.
|
||||
|
||||
The following example sets the id of a document to `1`:
|
||||
The following example sets the `_id` metadata field of a document to `1`:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -411,15 +435,20 @@ The following example sets the id of a document to `1`:
|
|||
--------------------------------------------------
|
||||
|
||||
The following metadata fields are accessible by a processor: `_index`, `_type`, `_id`, `_routing`, `_parent`,
|
||||
`_timestamp` and `_ttl`.
|
||||
`_timestamp`, and `_ttl`.
|
||||
|
||||
Beyond metadata fields and source fields, ingest also adds ingest metadata to documents being processed.
|
||||
[float]
|
||||
[[accessing-ingest-metadata]]
|
||||
=== Accessing Ingest Metadata Fields
|
||||
Beyond metadata fields and source fields, ingest also adds ingest metadata to the documents that it processes.
|
||||
These metadata properties are accessible under the `_ingest` key. Currently ingest adds the ingest timestamp
|
||||
under `_ingest.timestamp` key to the ingest metadata, which is the time ES received the index or bulk
|
||||
request to pre-process. But any processor is free to add more ingest related metadata to it. Ingest metadata is transient
|
||||
and is lost after a document has been processed by the pipeline and thus ingest metadata won't be indexed.
|
||||
under the `_ingest.timestamp` key of the ingest metadata. The ingest timestamp is the time when Elasticsearch
|
||||
received the index or bulk request to pre-process the document.
|
||||
|
||||
The following example adds a field with the name `received` and the value is the ingest timestamp:
|
||||
Any processor can add ingest-related metadata during document processing. Ingest metadata is transient
|
||||
and is lost after a document has been processed by the pipeline. Therefore, ingest metadata won't be indexed.
|
||||
|
||||
The following example adds a field with the name `received`. The value is the ingest timestamp:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -431,15 +460,18 @@ The following example adds a field with the name `received` and the value is the
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
As opposed to Elasticsearch metadata fields, the ingest metadata field name _ingest can be used as a valid field name
|
||||
in the source of a document. Use _source._ingest to refer to it, otherwise _ingest will be interpreted as ingest
|
||||
metadata fields.
|
||||
Unlike Elasticsearch metadata fields, the ingest metadata field name `_ingest` can be used as a valid field name
|
||||
in the source of a document. Use `_source._ingest` to refer to the field in the source document. Otherwise, `_ingest`
|
||||
will be interpreted as an ingest metadata field.
|
||||
|
||||
[float]
|
||||
[[accessing-template-fields]]
|
||||
=== Accessing Fields and Metafields in Templates
|
||||
A number of processor settings also support templating. Settings that support templating can have zero or more
|
||||
template snippets. A template snippet begins with `{{` and ends with `}}`.
|
||||
Accessing fields and metafields in templates is exactly the same as via regular processor field settings.
|
||||
|
||||
In this example a field by the name `field_c` is added and its value is a concatenation of
|
||||
The following example adds a field named `field_c`. Its value is a concatenation of
|
||||
the values of `field_a` and `field_b`.
|
||||
|
||||
[source,js]
|
||||
|
@ -452,8 +484,8 @@ the values of `field_a` and `field_b`.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
The following example changes the index a document is going to be indexed into. The index a document will be redirected
|
||||
to depends on the field in the source with name `geoip.country_iso_code`.
|
||||
The following example uses the value of the `geoip.country_iso_code` field in the source
|
||||
to set the index that the document will be indexed into:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -466,25 +498,25 @@ to depends on the field in the source with name `geoip.country_iso_code`.
|
|||
--------------------------------------------------
|
||||
|
||||
[[handling-failure-in-pipelines]]
|
||||
=== Handling Failure in Pipelines
|
||||
== Handling Failures in Pipelines
|
||||
|
||||
In its simplest case, pipelines describe a list of processors which
|
||||
are executed sequentially and processing halts at the first exception. This
|
||||
may not be desirable when failures are expected. For example, not all your logs
|
||||
may match a certain grok expression and you may wish to index such documents into
|
||||
a separate index.
|
||||
In its simplest use case, a pipeline defines a list of processors that
|
||||
are executed sequentially, and processing halts at the first exception. This
|
||||
behavior may not be desirable when failures are expected. For example, you may have logs
|
||||
that don't match the specified grok expression. Instead of halting execution, you may
|
||||
want to index such documents into a separate index.
|
||||
|
||||
To enable this behavior, you can utilize the `on_failure` parameter. `on_failure`
|
||||
To enable this behavior, you can use the `on_failure` parameter. The `on_failure` parameter
|
||||
defines a list of processors to be executed immediately following the failed processor.
|
||||
This parameter can be supplied at the pipeline level, as well as at the processor
|
||||
level. If a processor has an `on_failure` configuration option provided, whether
|
||||
it is empty or not, any exceptions that are thrown by it will be caught and the
|
||||
pipeline will continue executing the proceeding processors defined. Since further processors
|
||||
are defined within the scope of an `on_failure` statement, failure handling can be nested.
|
||||
You can specify this parameter at the pipeline level, as well as at the processor
|
||||
level. If a processor specifies an `on_failure` configuration, whether
|
||||
it is empty or not, any exceptions that are thrown by the processor are caught, and the
|
||||
pipeline continues executing the remaining processors. Because you can define further processors
|
||||
within the scope of an `on_failure` statement, you can nest failure handling.
|
||||
|
||||
Example: In the following example we define a pipeline that hopes to rename documents with
|
||||
a field named `foo` to `bar`. If the document does not contain the `foo` field, we
|
||||
go ahead and attach an error message within the document for later analysis within
|
||||
The following example defines a pipeline that renames the `foo` field in
|
||||
the processed document to `bar`. If the document does not contain the `foo` field, the processor
|
||||
attaches an error message to the document for later analysis within
|
||||
Elasticsearch.
|
||||
|
||||
[source,js]
|
||||
|
@ -510,8 +542,8 @@ Elasticsearch.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
Example: Here we define an `on_failure` block on a whole pipeline to change
|
||||
the index for which failed documents get sent.
|
||||
The following example defines an `on_failure` block on a whole pipeline to change
|
||||
the index to which failed documents get sent.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -529,15 +561,18 @@ the index for which failed documents get sent.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
[float]
|
||||
[[accessing-error-metadata]]
|
||||
=== Accessing Error Metadata From Processors Handling Exceptions
|
||||
|
||||
==== Accessing Error Metadata From Processors Handling Exceptions
|
||||
You may want to retrieve the actual error message that was thrown
|
||||
by a failed processor. To do so you can access metadata fields called
|
||||
`on_failure_message`, `on_failure_processor_type`, and `on_failure_processor_tag`. These fields are only accessible
|
||||
from within the context of an `on_failure` block.
|
||||
|
||||
Sometimes you may want to retrieve the actual error message that was thrown
|
||||
by a failed processor. To do so you can access metadata fields called
|
||||
`on_failure_message`, `on_failure_processor_type`, `on_failure_processor_tag`. These fields are only accessible
|
||||
from within the context of an `on_failure` block. Here is an updated version of
|
||||
our first example which leverages these fields to provide the error message instead
|
||||
of manually setting it.
|
||||
Here is an updated version of the example that you
|
||||
saw earlier. But instead of setting the error message manually, the example leverages the `on_failure_message`
|
||||
metadata field to provide the error message.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -562,6 +597,7 @@ of manually setting it.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
[[ingest-processors]]
|
||||
== Processors
|
||||
|
||||
All processors are defined in the following way within a pipeline definition:
|
||||
|
@ -575,19 +611,20 @@ All processors are defined in the following way within a pipeline definition:
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
Each processor defines its own configuration parameters, but all processors have
|
||||
Each processor defines its own configuration parameters, but all processors have
|
||||
the ability to declare `tag` and `on_failure` fields. These fields are optional.
|
||||
|
||||
A `tag` is simply a string identifier of the specific instantiation of a certain
|
||||
processor in a pipeline. The `tag` field does not affect any processor's behavior,
|
||||
processor in a pipeline. The `tag` field does not affect the processor's behavior,
|
||||
but is very useful for bookkeeping and tracing errors to specific processors.
|
||||
|
||||
See <<handling-failure-in-pipelines>> to learn more about the `on_failure` field and error handling in pipelines.
|
||||
|
||||
=== Append processor
|
||||
[[append-procesesor]]
|
||||
=== Append Processor
|
||||
Appends one or more values to an existing array if the field already exists and it is an array.
|
||||
Converts a scalar to an array and appends one or more values to it if the field exists and it is a scalar.
|
||||
Creates an array containing the provided values if the fields doesn't exist.
|
||||
Creates an array containing the provided values if the field doesn't exist.
|
||||
Accepts a single value or an array of values.
|
||||
|
||||
[[append-options]]
|
||||
|
@ -609,14 +646,15 @@ Accepts a single value or an array of values.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Convert processor
|
||||
Converts an existing field's value to a different type, like turning a string to an integer.
|
||||
[[convert-processor]]
|
||||
=== Convert Processor
|
||||
Converts an existing field's value to a different type, such as converting a string to an integer.
|
||||
If the field value is an array, all members will be converted.
|
||||
|
||||
The supported types include: `integer`, `float`, `string`, and `boolean`.
|
||||
|
||||
`boolean` will set the field to true if its string value is equal to `true` (ignore case), to
|
||||
false if its string value is equal to `false` (ignore case) and it will throw exception otherwise.
|
||||
Specifying `boolean` will set the field to true if its string value is equal to `true` (ignore case), to
|
||||
false if its string value is equal to `false` (ignore case), or it will throw an exception otherwise.
|
||||
|
||||
[[convert-options]]
|
||||
.Convert Options
|
||||
|
@ -637,12 +675,14 @@ false if its string value is equal to `false` (ignore case) and it will throw ex
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Date processor
|
||||
[[date-processor]]
|
||||
=== Date Processor
|
||||
|
||||
The date processor is used for parsing dates from fields, and then using that date or timestamp as the timestamp for that document.
|
||||
The date processor adds by default the parsed date as a new field called `@timestamp`, configurable by setting the `target_field`
|
||||
configuration parameter. Multiple date formats are supported as part of the same date processor definition. They will be used
|
||||
sequentially to attempt parsing the date field, in the same order they were defined as part of the processor definition.
|
||||
Parses dates from fields, and then uses the date or timestamp as the timestamp for the document.
|
||||
By default, the date processor adds the parsed date as a new field called `@timestamp`. You can specify a
|
||||
different field by setting the `target_field` configuration parameter. Multiple date formats are supported
|
||||
as part of the same date processor definition. They will be used sequentially to attempt parsing the date field,
|
||||
in the same order they were defined as part of the processor definition.
|
||||
|
||||
[[date-options]]
|
||||
.Date options
|
||||
|
@ -651,12 +691,12 @@ sequentially to attempt parsing the date field, in the same order they were defi
|
|||
| Name | Required | Default | Description
|
||||
| `match_field` | yes | - | The field to get the date from.
|
||||
| `target_field` | no | @timestamp | The field that will hold the parsed date.
|
||||
| `match_formats` | yes | - | Array of the expected date formats. Can be a joda pattern or one of the following formats: ISO8601, UNIX, UNIX_MS, TAI64N.
|
||||
| `match_formats` | yes | - | An array of the expected date formats. Can be a Joda pattern or one of the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N.
|
||||
| `timezone` | no | UTC | The timezone to use when parsing the date.
|
||||
| `locale` | no | ENGLISH | The locale to use when parsing the date, relevant when parsing month names or week days.
|
||||
|======
|
||||
|
||||
An example that adds the parsed date to the `timestamp` field based on the `initial_date` field:
|
||||
Here is an example that adds the parsed date to the `timestamp` field based on the `initial_date` field:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -675,9 +715,10 @@ An example that adds the parsed date to the `timestamp` field based on the `init
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Fail processor
|
||||
The Fail Processor is used to raise an exception. This is useful for when
|
||||
a user expects a pipeline to fail and wishes to relay a specific message
|
||||
[[fail-processor]]
|
||||
=== Fail Processor
|
||||
Raises an exception. This is useful for when
|
||||
you expect a pipeline to fail and want to relay a specific message
|
||||
to the requester.
|
||||
|
||||
[[fail-options]]
|
||||
|
@ -697,17 +738,20 @@ to the requester.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Foreach processor
|
||||
All processors can operate on elements inside an array, but if all elements of an array need to
|
||||
be processed in the same way defining a processor for each element becomes cumbersome and tricky
|
||||
because it is likely that the number of elements in an array are unknown. For this reason the `foreach`
|
||||
processor is exists. By specifying the field holding array elements and a list of processors that
|
||||
define what should happen to each element, array field can easily be preprocessed.
|
||||
[[foreach-processor]]
|
||||
=== Foreach Processor
|
||||
Processes elements in an array of unknown length.
|
||||
|
||||
Processors inside the foreach processor work in a different context and the only valid top level
|
||||
All processors can operate on elements inside an array, but if all elements of an array need to
|
||||
be processed in the same way, defining a processor for each element becomes cumbersome and tricky
|
||||
because it is likely that the number of elements in an array is unknown. For this reason the `foreach`
|
||||
processor exists. By specifying the field holding array elements and a list of processors that
|
||||
define what should happen to each element, array fields can easily be preprocessed.
|
||||
|
||||
Processors inside the foreach processor work in a different context, and the only valid top-level
|
||||
field is `_value`, which holds the array element value. Under this field other fields may exist.
|
||||
|
||||
If the `foreach` processor failed to process an element inside the array and no `on_failure` processor has been specified
|
||||
If the `foreach` processor fails to process an element inside the array, and no `on_failure` processor has been specified,
|
||||
then it aborts the execution and leaves the array unmodified.
|
||||
|
||||
[[foreach-options]]
|
||||
|
@ -755,7 +799,7 @@ Then the document will look like this after preprocessing:
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
Lets take a look at another example:
|
||||
Let's take a look at another example:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -773,8 +817,8 @@ Lets take a look at another example:
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
and in the case the `id` field needs to be removed
|
||||
then the following `foreach` processor can be used:
|
||||
In this case, the `id` field needs to be removed,
|
||||
so the following `foreach` processor is used:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -808,12 +852,12 @@ After preprocessing the result is:
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
Like on any processor `on_failure` processors can also be defined
|
||||
in processors that wrapped inside the `foreach` processor.
|
||||
As for any processor, you can define `on_failure` processors
|
||||
in processors that are wrapped inside the `foreach` processor.
|
||||
|
||||
For example the `id` field may not exist on all person objects and
|
||||
instead of failing the index request, the document will be send to
|
||||
the 'failure_index' index for later inspection:
|
||||
For example, the `id` field may not exist on all person objects.
|
||||
Instead of failing the index request, you can use an `on_failure`
|
||||
block to send the document to the 'failure_index' index for later inspection:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -839,14 +883,15 @@ the 'failure_index' index for later inspection:
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
In this example if the `remove` processor does fail then
|
||||
In this example, if the `remove` processor does fail, then
|
||||
the array elements that have been processed thus far will
|
||||
be updated.
|
||||
|
||||
[[grok-processor]]
|
||||
=== Grok Processor
|
||||
|
||||
The Grok Processor extracts structured fields out of a single text field within a document. You choose which field to
|
||||
extract matched fields from, as well as the Grok Pattern you expect will match. A Grok Pattern is like a regular
|
||||
Extracts structured fields out of a single text field within a document. You choose which field to
|
||||
extract matched fields from, as well as the grok pattern you expect will match. A grok pattern is like a regular
|
||||
expression that supports aliased expressions that can be reused.
|
||||
|
||||
This tool is perfect for syslog logs, apache and other webserver logs, mysql logs, and in general, any log format
|
||||
|
@ -858,6 +903,7 @@ Here, you can add your own custom grok pattern files with custom grok expression
|
|||
If you need help building patterns to match your logs, you will find the <http://grokdebug.herokuapp.com> and
|
||||
<http://grokconstructor.appspot.com/> applications quite useful!
|
||||
|
||||
[[grok-basics]]
|
||||
==== Grok Basics
|
||||
|
||||
Grok sits on top of regular expressions, so any regular expressions are valid in grok as well.
|
||||
|
@ -867,7 +913,7 @@ https://github.com/kkos/oniguruma/blob/master/doc/RE[on the Onigiruma site].
|
|||
Grok works by leveraging this regular expression language to allow naming existing patterns and combining them into more
|
||||
complex patterns that match your fields.
|
||||
|
||||
The syntax for re-using a grok pattern comes in three forms: `%{SYNTAX:SEMANTIC}`, `%{SYNTAX}`, `%{SYNTAX:SEMANTIC:TYPE}`.
|
||||
The syntax for reusing a grok pattern comes in three forms: `%{SYNTAX:SEMANTIC}`, `%{SYNTAX}`, `%{SYNTAX:SEMANTIC:TYPE}`.
|
||||
|
||||
The `SYNTAX` is the name of the pattern that will match your text. For example, `3.44` will be matched by the `NUMBER`
|
||||
pattern and `55.3.244.1` will be matched by the `IP` pattern. The syntax is how you match. `NUMBER` and `IP` are both
|
||||
|
@ -879,15 +925,14 @@ the `client` making a request.
|
|||
|
||||
The `TYPE` is the type you wish to cast your named field. `int` and `float` are currently the only types supported for coercion.
|
||||
|
||||
For example, here is a grok pattern that would match the above example given. We would like to match a text with the following
|
||||
contents:
|
||||
For example, you might want to match the following text:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
3.44 55.3.244.1
|
||||
--------------------------------------------------
|
||||
|
||||
We may know that the above message is a number followed by an IP-address. We can match this text with the following
|
||||
You may know that the message in the example is a number followed by an IP address. You can match this text by using the following
|
||||
Grok expression.
|
||||
|
||||
[source,js]
|
||||
|
@ -895,9 +940,10 @@ Grok expression.
|
|||
%{NUMBER:duration} %{IP:client}
|
||||
--------------------------------------------------
|
||||
|
||||
[[custom-patterns]]
|
||||
==== Custom Patterns and Pattern Files
|
||||
|
||||
The Grok Processor comes pre-packaged with a base set of pattern files. These patterns may not always have
|
||||
The Grok processor comes pre-packaged with a base set of pattern files. These patterns may not always have
|
||||
what you are looking for. These pattern files have a very basic format. Each line describes a named pattern with
|
||||
the following format:
|
||||
|
||||
|
@ -906,11 +952,11 @@ the following format:
|
|||
NAME ' '+ PATTERN '\n'
|
||||
--------------------------------------------------
|
||||
|
||||
You can add this pattern to an existing file, or add your own file in the patterns directory here: `$ES_HOME/config/ingest/grok/patterns`.
|
||||
The Ingest Plugin will pick up files in this directory to be loaded into the grok processor's known patterns. These patterns are loaded
|
||||
at startup, so you will need to do a restart your ingest node if you wish to update these files while running.
|
||||
You can add new patterns to an existing file, or add your own file in the patterns directory here: `$ES_HOME/config/ingest/grok/patterns`.
|
||||
Ingest node picks up files in this directory and loads the patterns into the grok processor's known patterns.
|
||||
These patterns are loaded at startup, so you need to restart your ingest node if you want to update these files.
|
||||
|
||||
Example snippet of pattern definitions found in the `grok-patterns` patterns file:
|
||||
Here is an example snippet of pattern definitions found in the `grok-patterns` patterns file:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -921,7 +967,8 @@ SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
|
|||
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
|
||||
--------------------------------------------------
|
||||
|
||||
==== Using Grok Processor in a Pipeline
|
||||
[[using-grok]]
|
||||
==== Using the Grok Processor in a Pipeline
|
||||
|
||||
[[grok-options]]
|
||||
.Grok Options
|
||||
|
@ -943,14 +990,14 @@ a document.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
The pattern for this could be
|
||||
The pattern for this could be:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}
|
||||
--------------------------------------------------
|
||||
|
||||
An example pipeline for processing the above document using Grok:
|
||||
Here is an example pipeline for processing the above document by using Grok:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -981,7 +1028,7 @@ This pipeline will insert these named captures as new fields within the document
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
An example of a pipeline specifying custom pattern definitions:
|
||||
Here is an example of a pipeline specifying custom pattern definitions:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -1002,7 +1049,8 @@ An example of a pipeline specifying custom pattern definitions:
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Gsub processor
|
||||
[[gsub-processor]]
|
||||
=== Gsub Processor
|
||||
Converts a string field by applying a regular expression and a replacement.
|
||||
If the field is not a string, the processor will throw an exception.
|
||||
|
||||
|
@ -1011,9 +1059,9 @@ If the field is not a string, the processor will throw an exception.
|
|||
[options="header"]
|
||||
|======
|
||||
| Name | Required | Default | Description
|
||||
| `field` | yes | - | The field apply the replacement for
|
||||
| `field` | yes | - | The field to apply the replacement to
|
||||
| `pattern` | yes | - | The pattern to be replaced
|
||||
| `replacement` | yes | - | The string to replace the matching patterns with.
|
||||
| `replacement` | yes | - | The string to replace the matching patterns with
|
||||
|======
|
||||
|
||||
[source,js]
|
||||
|
@ -1027,9 +1075,10 @@ If the field is not a string, the processor will throw an exception.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Join processor
|
||||
[[join-processor]]
|
||||
=== Join Processor
|
||||
Joins each element of an array into a single string using a separator character between each element.
|
||||
Throws error when the field is not an array.
|
||||
Throws an error when the field is not an array.
|
||||
|
||||
[[join-options]]
|
||||
.Join Options
|
||||
|
@ -1050,7 +1099,8 @@ Throws error when the field is not an array.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Lowercase processor
|
||||
[[lowercase-processor]]
|
||||
=== Lowercase Processor
|
||||
Converts a string to its lowercase equivalent.
|
||||
|
||||
[[lowercase-options]]
|
||||
|
@ -1058,7 +1108,7 @@ Converts a string to its lowercase equivalent.
|
|||
[options="header"]
|
||||
|======
|
||||
| Name | Required | Default | Description
|
||||
| `field` | yes | - | The field to lowercase
|
||||
| `field` | yes | - | The field to make lowercase
|
||||
|======
|
||||
|
||||
[source,js]
|
||||
|
@ -1070,8 +1120,9 @@ Converts a string to its lowercase equivalent.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Remove processor
|
||||
Removes an existing field. If the field doesn't exist, an exception will be thrown
|
||||
[[remove-processor]]
|
||||
=== Remove Processor
|
||||
Removes an existing field. If the field doesn't exist, an exception will be thrown.
|
||||
|
||||
[[remove-options]]
|
||||
.Remove Options
|
||||
|
@ -1090,9 +1141,9 @@ Removes an existing field. If the field doesn't exist, an exception will be thro
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Rename processor
|
||||
Renames an existing field. If the field doesn't exist, an exception will be thrown. Also, the new field
|
||||
name must not exist.
|
||||
[[rename-processor]]
|
||||
=== Rename Processor
|
||||
Renames an existing field. If the field doesn't exist or the new name is already used, an exception will be thrown.
|
||||
|
||||
[[rename-options]]
|
||||
.Rename Options
|
||||
|
@ -1113,7 +1164,8 @@ name must not exist.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Set processor
|
||||
[[set-processor]]
|
||||
=== Set Processor
|
||||
Sets one field and associates it with the specified value. If the field already exists,
|
||||
its value will be replaced with the provided one.
|
||||
|
||||
|
@ -1136,8 +1188,9 @@ its value will be replaced with the provided one.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Split processor
|
||||
Split a field to an array using a separator character. Only works on string fields.
|
||||
[[split-processor]]
|
||||
=== Split Processor
|
||||
Splits a field into an array using a separator character. Only works on string fields.
|
||||
|
||||
[[split-options]]
|
||||
.Split Options
|
||||
|
@ -1156,8 +1209,11 @@ Split a field to an array using a separator character. Only works on string fiel
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Trim processor
|
||||
Trims whitespace from field. NOTE: this only works on leading and trailing whitespaces.
|
||||
[[trim-processor]]
|
||||
=== Trim Processor
|
||||
Trims whitespace from field.
|
||||
|
||||
NOTE: This only works on leading and trailing whitespace.
|
||||
|
||||
[[trim-options]]
|
||||
.Trim Options
|
||||
|
@ -1176,7 +1232,8 @@ Trims whitespace from field. NOTE: this only works on leading and trailing white
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
=== Uppercase processor
|
||||
[[uppercase-processor]]
|
||||
=== Uppercase Processor
|
||||
Converts a string to its uppercase equivalent.
|
||||
|
||||
[[uppercase-options]]
|
||||
|
@ -1184,7 +1241,7 @@ Converts a string to its uppercase equivalent.
|
|||
[options="header"]
|
||||
|======
|
||||
| Name | Required | Default | Description
|
||||
| `field` | yes | - | The field to uppercase
|
||||
| `field` | yes | - | The field to make uppercase
|
||||
|======
|
||||
|
||||
[source,js]
|
||||
|
|
|
@ -161,7 +161,7 @@ Individual fields can be included or excluded from the `_all` field with the
|
|||
[[all-field-and-boosting]]
|
||||
==== Index boosting and the `_all` field
|
||||
|
||||
Individual fields can be _boosted_ at index time, with the <<index-boost,`boost`>>
|
||||
Individual fields can be _boosted_ at index time, with the <<mapping-boost,`boost`>>
|
||||
parameter. The `_all` field takes these boosts into account:
|
||||
|
||||
[source,js]
|
||||
|
|
|
@ -8,7 +8,7 @@ parameters that are used by <<mapping-types,field mappings>>:
|
|||
The following mapping parameters are common to some or all field datatypes:
|
||||
|
||||
* <<analyzer,`analyzer`>>
|
||||
* <<index-boost,`boost`>>
|
||||
* <<mapping-boost,`boost`>>
|
||||
* <<coerce,`coerce`>>
|
||||
* <<copy-to,`copy_to`>>
|
||||
* <<doc-values,`doc_values`>>
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
[[index-boost]]
|
||||
[[mapping-boost]]
|
||||
=== `boost`
|
||||
|
||||
Individual fields can be _boosted_ -- count more towards the relevance score
|
||||
-- at index time, with the `boost` parameter as follows:
|
||||
Individual fields can be _boosted_ automatically -- count more towards the relevance score
|
||||
-- at query time, with the `boost` parameter as follows:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
|
@ -28,10 +28,45 @@ PUT my_index
|
|||
<1> Matches on the `title` field will have twice the weight as those on the
|
||||
`content` field, which has the default `boost` of `1.0`.
|
||||
|
||||
Note that a `title` field will usually be shorter than a `content` field. The
|
||||
default relevance calculation takes field length into account, so a short
|
||||
`title` field will have a higher natural boost than a long `content` field.
|
||||
NOTE: The boost is applied only for term queries (prefix, range and fuzzy queries are not _boosted_).
|
||||
|
||||
You can achieve the same effect by using the boost parameter directly in the query, for instance the following query (with field time boost):
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
{
|
||||
"match" : {
|
||||
"title": {
|
||||
"query": "quick brown fox"
|
||||
}
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
|
||||
is equivalent to:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
{
|
||||
"match" : {
|
||||
"title": {
|
||||
"query": "quick brown fox",
|
||||
"boost": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
// AUTOSENSE
|
||||
|
||||
|
||||
The boost is also applied when it is copied with the
|
||||
value in the <<mapping-all-field,`_all`>> field. This means that, when
|
||||
querying the `_all` field, words that originated from the `title` field will
|
||||
have a higher score than words that originated in the `content` field.
|
||||
This functionality comes at a cost: queries on the `_all` field are slower
|
||||
when field boosting is used.
|
||||
|
||||
deprecated[5.0.0, index time boost is deprecated. Instead, the field mapping boost is applied at query time. For indices created before 5.0.0 the boost will still be applied at index time.]
|
||||
[WARNING]
|
||||
.Why index time boosting is a bad idea
|
||||
==================================================
|
||||
|
@ -48,12 +83,4 @@ We advise against using index time boosting for the following reasons:
|
|||
byte. This reduces the resolution of the field length normalization factor
|
||||
which can lead to lower quality relevance calculations.
|
||||
|
||||
==================================================
|
||||
|
||||
The only advantage that index time boosting has is that it is copied with the
|
||||
value into the <<mapping-all-field,`_all`>> field. This means that, when
|
||||
querying the `_all` field, words that originated from the `title` field will
|
||||
have a higher score than words that originated in the `content` field.
|
||||
This functionality comes at a cost: queries on the `_all` field are slower
|
||||
when index-time boosting is used.
|
||||
|
||||
==================================================
|
|
@ -1,10 +1,8 @@
|
|||
[[norms]]
|
||||
=== `norms`
|
||||
|
||||
Norms store various normalization factors -- a number to represent the
|
||||
relative field length and the <<index-boost,index time `boost`>> setting --
|
||||
that are later used at query time in order to compute the score of a document
|
||||
relatively to a query.
|
||||
Norms store various normalization factors that are later used at query time
|
||||
in order to compute the score of a document relatively to a query.
|
||||
|
||||
Although useful for scoring, norms also require quite a lot of memory
|
||||
(typically in the order of one byte per document per field in your index, even
|
||||
|
|
|
@ -91,9 +91,9 @@ The following parameters are accepted by `boolean` fields:
|
|||
|
||||
[horizontal]
|
||||
|
||||
<<index-boost,`boost`>>::
|
||||
<<mapping-boost,`boost`>>::
|
||||
|
||||
Field-level index time boosting. Accepts a floating point number, defaults
|
||||
Mapping field-level query time boosting. Accepts a floating point number, defaults
|
||||
to `1.0`.
|
||||
|
||||
<<doc-values,`doc_values`>>::
|
||||
|
|
|
@ -90,9 +90,9 @@ The following parameters are accepted by `date` fields:
|
|||
|
||||
[horizontal]
|
||||
|
||||
<<index-boost,`boost`>>::
|
||||
<<mapping-boost,`boost`>>::
|
||||
|
||||
Field-level index time boosting. Accepts a floating point number, defaults
|
||||
Mapping field-level query time boosting. Accepts a floating point number, defaults
|
||||
to `1.0`.
|
||||
|
||||
<<doc-values,`doc_values`>>::
|
||||
|
|
|
@ -47,9 +47,9 @@ The following parameters are accepted by `ip` fields:
|
|||
|
||||
[horizontal]
|
||||
|
||||
<<index-boost,`boost`>>::
|
||||
<<mapping-boost,`boost`>>::
|
||||
|
||||
Field-level index time boosting. Accepts a floating point number, defaults
|
||||
Mapping field-level query time boosting. Accepts a floating point number, defaults
|
||||
to `1.0`.
|
||||
|
||||
<<doc-values,`doc_values`>>::
|
||||
|
|
|
@ -45,9 +45,9 @@ The following parameters are accepted by numeric types:
|
|||
Try to convert strings to numbers and truncate fractions for integers.
|
||||
Accepts `true` (default) and `false`.
|
||||
|
||||
<<index-boost,`boost`>>::
|
||||
<<mapping-boost,`boost`>>::
|
||||
|
||||
Field-level index time boosting. Accepts a floating point number, defaults
|
||||
Mapping field-level query time boosting. Accepts a floating point number, defaults
|
||||
to `1.0`.
|
||||
|
||||
<<doc-values,`doc_values`>>::
|
||||
|
|
|
@ -75,9 +75,9 @@ The following parameters are accepted by `string` fields:
|
|||
Defaults to the default index analyzer, or the
|
||||
<<analysis-standard-analyzer,`standard` analyzer>>.
|
||||
|
||||
<<index-boost,`boost`>>::
|
||||
<<mapping-boost,`boost`>>::
|
||||
|
||||
Field-level index time boosting. Accepts a floating point number, defaults
|
||||
Mapping field-level query time boosting. Accepts a floating point number, defaults
|
||||
to `1.0`.
|
||||
|
||||
<<doc-values,`doc_values`>>::
|
||||
|
|
|
@ -68,9 +68,9 @@ The following parameters are accepted by `token_count` fields:
|
|||
value. Required. For best performance, use an analyzer without token
|
||||
filters.
|
||||
|
||||
<<index-boost,`boost`>>::
|
||||
<<mapping-boost,`boost`>>::
|
||||
|
||||
Field-level index time boosting. Accepts a floating point number, defaults
|
||||
Mapping field-level query time boosting. Accepts a floating point number, defaults
|
||||
to `1.0`.
|
||||
|
||||
<<doc-values,`doc_values`>>::
|
||||
|
|
|
@ -449,3 +449,8 @@ The docs for the `nested` field datatype have moved to <<nested>>.
|
|||
|
||||
Warmers have been removed. There have been significant improvements to the
|
||||
index that make warmers not necessary anymore.
|
||||
|
||||
[role="exclude",id="index-boost"]
|
||||
=== Index time boosting
|
||||
|
||||
The index time boost mapping has been replaced with query time boost (see <<mapping-boost>>).
|
||||
|
|
|
@ -60,8 +60,8 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
|
|||
static {
|
||||
ObjectParser.Parser<SearchRequest, ReindexParseContext> sourceParser = (parser, search, context) -> {
|
||||
/*
|
||||
* Extract the parameters that we need from the parser. We could do
|
||||
* away with this hack when search source has an ObjectParser.
|
||||
* Extract the parameters that we need from the source sent to the parser. We could do away with this hack when search source
|
||||
* has an ObjectParser.
|
||||
*/
|
||||
Map<String, Object> source = parser.map();
|
||||
String[] indices = extractStringArray(source, "index");
|
||||
|
@ -84,6 +84,7 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
|
|||
destParser.declareString(IndexRequest::type, new ParseField("type"));
|
||||
destParser.declareString(IndexRequest::routing, new ParseField("routing"));
|
||||
destParser.declareString(IndexRequest::opType, new ParseField("opType"));
|
||||
destParser.declareString(IndexRequest::setPipeline, new ParseField("pipeline"));
|
||||
destParser.declareString((s, i) -> s.versionType(VersionType.fromString(i)), new ParseField("versionType"));
|
||||
|
||||
// These exist just so the user can get a nice validation error:
|
||||
|
|
|
@ -105,8 +105,10 @@ public class RestUpdateByQueryAction extends
|
|||
parseCommon(internalRequest, request);
|
||||
|
||||
internalRequest.setSize(internalRequest.getSearchRequest().source().size());
|
||||
internalRequest.setPipeline(request.param("pipeline"));
|
||||
internalRequest.getSearchRequest().source().size(request.paramAsInt("scroll_size", scrollSize));
|
||||
|
||||
|
||||
execute(request, internalRequest, channel);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -158,6 +158,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
|
|||
index.timestamp(mainRequest.getDestination().timestamp());
|
||||
index.ttl(mainRequest.getDestination().ttl());
|
||||
index.contentType(mainRequest.getDestination().getContentType());
|
||||
index.setPipeline(mainRequest.getDestination().getPipeline());
|
||||
// OpType is synthesized from version so it is handled when we copy version above.
|
||||
|
||||
return index;
|
||||
|
|
|
@ -90,6 +90,7 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
|
|||
index.source(doc.sourceRef());
|
||||
index.versionType(VersionType.INTERNAL);
|
||||
index.version(doc.version());
|
||||
index.setPipeline(mainRequest.getPipeline());
|
||||
return index;
|
||||
}
|
||||
|
||||
|
|
|
@ -26,6 +26,11 @@ import org.elasticsearch.action.search.SearchRequest;
|
|||
* locations or IDs.
|
||||
*/
|
||||
public class UpdateByQueryRequest extends AbstractBulkIndexByScrollRequest<UpdateByQueryRequest> {
|
||||
/**
|
||||
* Ingest pipeline to set on index requests made by this action.
|
||||
*/
|
||||
private String pipeline;
|
||||
|
||||
public UpdateByQueryRequest() {
|
||||
}
|
||||
|
||||
|
@ -33,6 +38,20 @@ public class UpdateByQueryRequest extends AbstractBulkIndexByScrollRequest<Updat
|
|||
super(search);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the ingest pipeline to set on index requests made by this action.
|
||||
*/
|
||||
public void setPipeline(String pipeline) {
|
||||
this.pipeline = pipeline;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingest pipeline to set on index requests made by this action.
|
||||
*/
|
||||
public String getPipeline() {
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected UpdateByQueryRequest self() {
|
||||
return this;
|
||||
|
|
|
@ -0,0 +1,46 @@
|
|||
---
|
||||
"Modify a document":
|
||||
- do:
|
||||
ingest.put_pipeline:
|
||||
id: "test_ingest"
|
||||
body: >
|
||||
{
|
||||
"description": "tests reindex with ingest",
|
||||
"processors": [
|
||||
{
|
||||
"append" : {
|
||||
"field" : "new_field",
|
||||
"value": "cat"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
- do:
|
||||
index:
|
||||
index: twitter
|
||||
type: tweet
|
||||
id: 1
|
||||
body: { "user": "kimchy" }
|
||||
- do:
|
||||
indices.refresh: {}
|
||||
|
||||
- do:
|
||||
reindex:
|
||||
refresh: true
|
||||
body:
|
||||
source:
|
||||
index: twitter
|
||||
dest:
|
||||
index: new_twitter
|
||||
pipeline: test_ingest
|
||||
- match: {created: 1}
|
||||
- match: {noops: 0}
|
||||
|
||||
- do:
|
||||
search:
|
||||
index: new_twitter
|
||||
body:
|
||||
query:
|
||||
match:
|
||||
new_field: cat
|
||||
- match: { hits.total: 1 }
|
|
@ -0,0 +1,42 @@
|
|||
---
|
||||
"Update a document using update-by-query":
|
||||
- do:
|
||||
ingest.put_pipeline:
|
||||
id: "test_ingest"
|
||||
body: >
|
||||
{
|
||||
"description": "tests reindex with ingest",
|
||||
"processors": [
|
||||
{
|
||||
"append" : {
|
||||
"field" : "new_field",
|
||||
"value": "cat"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
- do:
|
||||
index:
|
||||
index: twitter
|
||||
type: tweet
|
||||
id: 1
|
||||
body: { "user": "kimchy" }
|
||||
- do:
|
||||
indices.refresh: {}
|
||||
|
||||
- do:
|
||||
update-by-query:
|
||||
index: twitter
|
||||
refresh: true
|
||||
pipeline: test_ingest
|
||||
- match: {updated: 1}
|
||||
- match: {noops: 0}
|
||||
|
||||
- do:
|
||||
search:
|
||||
index: twitter
|
||||
body:
|
||||
query:
|
||||
match:
|
||||
new_field: cat
|
||||
- match: { hits.total: 1 }
|
|
@ -19,12 +19,19 @@
|
|||
|
||||
package org.elasticsearch.mapper.attachments;
|
||||
|
||||
import org.elasticsearch.common.logging.DeprecationLogger;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.ESLoggerFactory;
|
||||
import org.elasticsearch.common.settings.SettingsModule;
|
||||
import org.elasticsearch.indices.IndicesModule;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
|
||||
public class MapperAttachmentsPlugin extends Plugin {
|
||||
|
||||
|
||||
private static ESLogger logger = ESLoggerFactory.getLogger("mapper.attachment");
|
||||
private static DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "mapper-attachments";
|
||||
|
@ -36,6 +43,7 @@ public class MapperAttachmentsPlugin extends Plugin {
|
|||
}
|
||||
|
||||
public void onModule(SettingsModule settingsModule) {
|
||||
deprecationLogger.deprecated("[mapper-attachments] plugin has been deprecated and will be replaced by [ingest-attachment] plugin.");
|
||||
settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING);
|
||||
settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_IGNORE_ERRORS_SETTING);
|
||||
settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_INDEXED_CHARS_SETTING);
|
||||
|
|
|
@ -24,3 +24,9 @@ dependencies {
|
|||
testCompile project(path: ':plugins:ingest-geoip', configuration: 'runtime')
|
||||
testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
|
||||
}
|
||||
|
||||
integTest {
|
||||
cluster {
|
||||
plugin 'ingest-geoip', project(':plugins:ingest-geoip')
|
||||
}
|
||||
}
|
|
@ -1,220 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.ingest;
|
||||
|
||||
import com.maxmind.geoip2.DatabaseReader;
|
||||
import org.elasticsearch.common.bytes.BytesArray;
|
||||
import org.elasticsearch.common.collect.HppcMaps;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.common.xcontent.XContentHelper;
|
||||
import org.elasticsearch.common.xcontent.support.XContentMapValues;
|
||||
import org.elasticsearch.ingest.core.CompoundProcessor;
|
||||
import org.elasticsearch.ingest.core.IngestDocument;
|
||||
import org.elasticsearch.ingest.core.Pipeline;
|
||||
import org.elasticsearch.ingest.core.Processor;
|
||||
import org.elasticsearch.ingest.geoip.GeoIpProcessor;
|
||||
import org.elasticsearch.ingest.geoip.IngestGeoIpPlugin;
|
||||
import org.elasticsearch.ingest.grok.GrokProcessor;
|
||||
import org.elasticsearch.ingest.grok.IngestGrokPlugin;
|
||||
import org.elasticsearch.ingest.processor.AppendProcessor;
|
||||
import org.elasticsearch.ingest.processor.ConvertProcessor;
|
||||
import org.elasticsearch.ingest.processor.DateProcessor;
|
||||
import org.elasticsearch.ingest.processor.ForEachProcessor;
|
||||
import org.elasticsearch.ingest.processor.LowercaseProcessor;
|
||||
import org.elasticsearch.ingest.processor.RemoveProcessor;
|
||||
import org.elasticsearch.ingest.processor.RenameProcessor;
|
||||
import org.elasticsearch.ingest.processor.SplitProcessor;
|
||||
import org.elasticsearch.ingest.processor.TrimProcessor;
|
||||
import org.elasticsearch.ingest.processor.UppercaseProcessor;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.elasticsearch.test.StreamsUtils;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.notNullValue;
|
||||
import static org.hamcrest.Matchers.nullValue;
|
||||
|
||||
public class CombineProcessorsTests extends ESTestCase {
|
||||
|
||||
private static final String LOG = "70.193.17.92 - - [08/Sep/2014:02:54:42 +0000] \"GET /presentations/logstash-scale11x/images/ahhh___rage_face_by_samusmmx-d5g5zap.png HTTP/1.1\" 200 175208 \"http://mobile.rivals.com/board_posts.asp?SID=880&mid=198829575&fid=2208&tid=198829575&Team=&TeamId=&SiteId=\" \"Mozilla/5.0 (Linux; Android 4.2.2; VS980 4G Build/JDQ39B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.135 Mobile Safari/537.36\"";
|
||||
|
||||
public void testLogging() throws Exception {
|
||||
Path configDir = createTempDir();
|
||||
Path geoIpConfigDir = configDir.resolve("ingest-geoip");
|
||||
Files.createDirectories(geoIpConfigDir);
|
||||
Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb")), geoIpConfigDir.resolve("GeoLite2-City.mmdb"));
|
||||
Map<String, DatabaseReader> databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpConfigDir);
|
||||
|
||||
Map<String, Object> config = new HashMap<>();
|
||||
config.put("field", "log");
|
||||
config.put("pattern", "%{COMBINEDAPACHELOG}");
|
||||
Processor processor1 = new GrokProcessor.Factory(IngestGrokPlugin.loadBuiltinPatterns()).doCreate(null, config);
|
||||
config = new HashMap<>();
|
||||
config.put("field", "response");
|
||||
config.put("type", "integer");
|
||||
Processor processor2 = new ConvertProcessor.Factory().create(config);
|
||||
config = new HashMap<>();
|
||||
config.put("field", "bytes");
|
||||
config.put("type", "integer");
|
||||
Processor processor3 = new ConvertProcessor.Factory().create(config);
|
||||
config = new HashMap<>();
|
||||
config.put("match_field", "timestamp");
|
||||
config.put("target_field", "timestamp");
|
||||
config.put("match_formats", Arrays.asList("dd/MMM/YYYY:HH:mm:ss Z"));
|
||||
Processor processor4 = new DateProcessor.Factory().create(config);
|
||||
config = new HashMap<>();
|
||||
config.put("source_field", "clientip");
|
||||
Processor processor5 = new GeoIpProcessor.Factory(databaseReaders).create(config);
|
||||
|
||||
Pipeline pipeline = new Pipeline("_id", "_description", new CompoundProcessor(processor1, processor2, processor3, processor4, processor5));
|
||||
|
||||
Map<String, Object> source = new HashMap<>();
|
||||
source.put("log", LOG);
|
||||
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null, source);
|
||||
pipeline.execute(document);
|
||||
|
||||
assertThat(document.getSourceAndMetadata().size(), equalTo(17));
|
||||
assertThat(document.getSourceAndMetadata().get("request"), equalTo("/presentations/logstash-scale11x/images/ahhh___rage_face_by_samusmmx-d5g5zap.png"));
|
||||
assertThat(document.getSourceAndMetadata().get("agent"), equalTo("\"Mozilla/5.0 (Linux; Android 4.2.2; VS980 4G Build/JDQ39B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.135 Mobile Safari/537.36\""));
|
||||
assertThat(document.getSourceAndMetadata().get("auth"), equalTo("-"));
|
||||
assertThat(document.getSourceAndMetadata().get("ident"), equalTo("-"));
|
||||
assertThat(document.getSourceAndMetadata().get("verb"), equalTo("GET"));
|
||||
assertThat(document.getSourceAndMetadata().get("referrer"), equalTo("\"http://mobile.rivals.com/board_posts.asp?SID=880&mid=198829575&fid=2208&tid=198829575&Team=&TeamId=&SiteId=\""));
|
||||
assertThat(document.getSourceAndMetadata().get("response"), equalTo(200));
|
||||
assertThat(document.getSourceAndMetadata().get("bytes"), equalTo(175208));
|
||||
assertThat(document.getSourceAndMetadata().get("clientip"), equalTo("70.193.17.92"));
|
||||
assertThat(document.getSourceAndMetadata().get("httpversion"), equalTo("1.1"));
|
||||
assertThat(document.getSourceAndMetadata().get("rawrequest"), nullValue());
|
||||
assertThat(document.getSourceAndMetadata().get("timestamp"), equalTo("2014-09-08T02:54:42.000Z"));
|
||||
Map<String, Object> geoInfo = (Map<String, Object>) document.getSourceAndMetadata().get("geoip");
|
||||
assertThat(geoInfo.size(), equalTo(5));
|
||||
assertThat(geoInfo.get("continent_name"), equalTo("North America"));
|
||||
assertThat(geoInfo.get("city_name"), equalTo("Charlotte"));
|
||||
assertThat(geoInfo.get("country_iso_code"), equalTo("US"));
|
||||
assertThat(geoInfo.get("region_name"), equalTo("North Carolina"));
|
||||
assertThat(geoInfo.get("location"), notNullValue());
|
||||
}
|
||||
|
||||
private static final String PERSON = "{\n" +
|
||||
" \"age\": 33,\n" +
|
||||
" \"eyeColor\": \"brown\",\n" +
|
||||
" \"name\": \"Miranda Goodwin\",\n" +
|
||||
" \"gender\": \"male\",\n" +
|
||||
" \"company\": \"ATGEN\",\n" +
|
||||
" \"email\": \"mirandagoodwin@atgen.com\",\n" +
|
||||
" \"phone\": \"+1 (914) 489-3656\",\n" +
|
||||
" \"address\": \"713 Bartlett Place, Accoville, Puerto Rico, 9221\",\n" +
|
||||
" \"registered\": \"2014-11-23T08:34:21 -01:00\",\n" +
|
||||
" \"tags\": [\n" +
|
||||
" \"ex\",\n" +
|
||||
" \"do\",\n" +
|
||||
" \"occaecat\",\n" +
|
||||
" \"reprehenderit\",\n" +
|
||||
" \"anim\",\n" +
|
||||
" \"laboris\",\n" +
|
||||
" \"cillum\"\n" +
|
||||
" ],\n" +
|
||||
" \"friends\": [\n" +
|
||||
" {\n" +
|
||||
" \"id\": 0,\n" +
|
||||
" \"name\": \"Wendi Odonnell\"\n" +
|
||||
" },\n" +
|
||||
" {\n" +
|
||||
" \"id\": 1,\n" +
|
||||
" \"name\": \"Mayra Boyd\"\n" +
|
||||
" },\n" +
|
||||
" {\n" +
|
||||
" \"id\": 2,\n" +
|
||||
" \"name\": \"Lee Gonzalez\"\n" +
|
||||
" }\n" +
|
||||
" ]\n" +
|
||||
" }";
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testMutate() throws Exception {
|
||||
ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
|
||||
builder.registerProcessor("remove", (templateService, registry) -> new RemoveProcessor.Factory(templateService));
|
||||
builder.registerProcessor("trim", (templateService, registry) -> new TrimProcessor.Factory());
|
||||
ProcessorsRegistry registry = builder.build(TestTemplateService.instance());
|
||||
|
||||
Map<String, Object> config = new HashMap<>();
|
||||
config.put("field", "friends");
|
||||
Map<String, Object> removeConfig = new HashMap<>();
|
||||
removeConfig.put("field", "_value.id");
|
||||
config.put("processors", Collections.singletonList(Collections.singletonMap("remove", removeConfig)));
|
||||
ForEachProcessor processor1 = new ForEachProcessor.Factory(registry).create(config);
|
||||
config = new HashMap<>();
|
||||
config.put("field", "tags");
|
||||
config.put("value", "new_value");
|
||||
AppendProcessor processor2 = new AppendProcessor.Factory(TestTemplateService.instance()).create(config);
|
||||
config = new HashMap<>();
|
||||
config.put("field", "address");
|
||||
config.put("separator", ",");
|
||||
SplitProcessor processor3 = new SplitProcessor.Factory().create(config);
|
||||
config = new HashMap<>();
|
||||
config.put("field", "address");
|
||||
Map<String, Object> trimConfig = new HashMap<>();
|
||||
trimConfig.put("field", "_value");
|
||||
config.put("processors", Collections.singletonList(Collections.singletonMap("trim", trimConfig)));
|
||||
ForEachProcessor processor4 = new ForEachProcessor.Factory(registry).create(config);
|
||||
config = new HashMap<>();
|
||||
config.put("field", "company");
|
||||
LowercaseProcessor processor5 = new LowercaseProcessor.Factory().create(config);
|
||||
config = new HashMap<>();
|
||||
config.put("field", "gender");
|
||||
UppercaseProcessor processor6 = new UppercaseProcessor.Factory().create(config);
|
||||
config = new HashMap<>();
|
||||
config.put("field", "eyeColor");
|
||||
config.put("to", "eye_color");
|
||||
RenameProcessor processor7 = new RenameProcessor.Factory().create(config);
|
||||
Pipeline pipeline = new Pipeline("_id", "_description", new CompoundProcessor(
|
||||
processor1, processor2, processor3, processor4, processor5, processor6, processor7
|
||||
));
|
||||
|
||||
Map<String, Object> source = XContentHelper.createParser(new BytesArray(PERSON)).map();
|
||||
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null, source);
|
||||
pipeline.execute(document);
|
||||
|
||||
assertThat(((List<Map<String, Object>>) document.getSourceAndMetadata().get("friends")).get(0).get("id"), nullValue());
|
||||
assertThat(((List<Map<String, Object>>) document.getSourceAndMetadata().get("friends")).get(1).get("id"), nullValue());
|
||||
assertThat(((List<Map<String, Object>>) document.getSourceAndMetadata().get("friends")).get(2).get("id"), nullValue());
|
||||
assertThat(document.getFieldValue("tags.7", String.class), equalTo("new_value"));
|
||||
|
||||
List<String> addressDetails = document.getFieldValue("address", List.class);
|
||||
assertThat(addressDetails.size(), equalTo(4));
|
||||
assertThat(addressDetails.get(0), equalTo("713 Bartlett Place"));
|
||||
assertThat(addressDetails.get(1), equalTo("Accoville"));
|
||||
assertThat(addressDetails.get(2), equalTo("Puerto Rico"));
|
||||
assertThat(addressDetails.get(3), equalTo("9221"));
|
||||
|
||||
assertThat(document.getSourceAndMetadata().get("company"), equalTo("atgen"));
|
||||
assertThat(document.getSourceAndMetadata().get("gender"), equalTo("MALE"));
|
||||
assertThat(document.getSourceAndMetadata().get("eye_color"), equalTo("brown"));
|
||||
}
|
||||
|
||||
}
|
|
@ -45,6 +45,7 @@ public class ValueSourceMustacheIT extends AbstractMustacheTestCase {
|
|||
|
||||
valueSource = ValueSource.wrap(Arrays.asList("_value", "{{field1}}"), templateService);
|
||||
assertThat(valueSource, instanceOf(ValueSource.ListValue.class));
|
||||
@SuppressWarnings("unchecked")
|
||||
List<String> result = (List<String>) valueSource.copyAndResolve(model);
|
||||
assertThat(result.size(), equalTo(2));
|
||||
assertThat(result.get(0), equalTo("_value"));
|
||||
|
@ -56,6 +57,7 @@ public class ValueSourceMustacheIT extends AbstractMustacheTestCase {
|
|||
map.put("field4", "_value");
|
||||
valueSource = ValueSource.wrap(map, templateService);
|
||||
assertThat(valueSource, instanceOf(ValueSource.MapValue.class));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> resultMap = (Map<String, Object>) valueSource.copyAndResolve(model);
|
||||
assertThat(resultMap.size(), equalTo(3));
|
||||
assertThat(resultMap.get("field1"), equalTo("value1"));
|
||||
|
|
|
@ -0,0 +1,199 @@
|
|||
---
|
||||
"Test logging":
|
||||
- do:
|
||||
ingest.put_pipeline:
|
||||
id: "_id"
|
||||
body: >
|
||||
{
|
||||
"processors": [
|
||||
{
|
||||
"grok" : {
|
||||
"field" : "log",
|
||||
"pattern": "%{COMBINEDAPACHELOG}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert" : {
|
||||
"field" : "response",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"convert" : {
|
||||
"field" : "bytes",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date" : {
|
||||
"match_field" : "timestamp",
|
||||
"target_field" : "timestamp",
|
||||
"match_formats" : ["dd/MMM/YYYY:HH:mm:ss Z"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"geoip" : {
|
||||
"source_field" : "clientip"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
- match: { acknowledged: true }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: test
|
||||
type: test
|
||||
id: 1
|
||||
pipeline: "_id"
|
||||
body: {
|
||||
log: "70.193.17.92 - - [08/Sep/2014:02:54:42 +0000] \"GET /presentations/logstash-scale11x/images/ahhh___rage_face_by_samusmmx-d5g5zap.png HTTP/1.1\" 200 175208 \"http://mobile.rivals.com/board_posts.asp?SID=880&mid=198829575&fid=2208&tid=198829575&Team=&TeamId=&SiteId=\" \"Mozilla/5.0 (Linux; Android 4.2.2; VS980 4G Build/JDQ39B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.135 Mobile Safari/537.36\""
|
||||
}
|
||||
|
||||
- do:
|
||||
get:
|
||||
index: test
|
||||
type: test
|
||||
id: 1
|
||||
- length: { _source: 14 }
|
||||
- match: { _source.request: "/presentations/logstash-scale11x/images/ahhh___rage_face_by_samusmmx-d5g5zap.png" }
|
||||
- match: { _source.agent: "\"Mozilla/5.0 (Linux; Android 4.2.2; VS980 4G Build/JDQ39B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.135 Mobile Safari/537.36\"" }
|
||||
- match: { _source.auth: "-" }
|
||||
- match: { _source.verb: "GET" }
|
||||
- match: { _source.referrer: "\"http://mobile.rivals.com/board_posts.asp?SID=880&mid=198829575&fid=2208&tid=198829575&Team=&TeamId=&SiteId=\"" }
|
||||
- match: { _source.response: 200 }
|
||||
- match: { _source.bytes: 175208 }
|
||||
- match: { _source.clientip: "70.193.17.92" }
|
||||
- match: { _source.httpversion: "1.1" }
|
||||
- match: { _source.timestamp: "2014-09-08T02:54:42.000Z" }
|
||||
- match: { _source.geoip.continent_name: "North America" }
|
||||
- match: { _source.geoip.city_name: "Charlotte" }
|
||||
- match: { _source.geoip.country_iso_code: "US" }
|
||||
- match: { _source.geoip.region_name: "North Carolina" }
|
||||
|
||||
---
|
||||
"Test mutate":
|
||||
- do:
|
||||
ingest.put_pipeline:
|
||||
id: "_id"
|
||||
body: >
|
||||
{
|
||||
"processors": [
|
||||
{
|
||||
"foreach" : {
|
||||
"field" : "friends",
|
||||
"processors" : [
|
||||
{
|
||||
"remove" : {
|
||||
"field" : "_value.id"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"append" : {
|
||||
"field" : "tags",
|
||||
"value": "new_value"
|
||||
}
|
||||
},
|
||||
{
|
||||
"split" : {
|
||||
"field" : "address",
|
||||
"separator": ","
|
||||
}
|
||||
},
|
||||
{
|
||||
"foreach" : {
|
||||
"field" : "address",
|
||||
"processors" : [
|
||||
{
|
||||
"trim" : {
|
||||
"field" : "_value"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"lowercase" : {
|
||||
"field" : "company"
|
||||
}
|
||||
},
|
||||
{
|
||||
"uppercase" : {
|
||||
"field" : "gender"
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename" : {
|
||||
"field" : "eyeColor",
|
||||
"to" : "eye_color"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
- match: { acknowledged: true }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: test
|
||||
type: test
|
||||
id: 1
|
||||
pipeline: "_id"
|
||||
body: {
|
||||
"age" : 33,
|
||||
"eyeColor" : "brown",
|
||||
"name" : "Miranda Goodwin",
|
||||
"gender" : "male",
|
||||
"company": "ATGEN",
|
||||
"email" : "mirandagoodwin@atgen.com",
|
||||
"phone": "+1 (914) 489-3656",
|
||||
"address" : "713 Bartlett Place, Accoville, Puerto Rico, 9221",
|
||||
"registered": "2014-11-23T08:34:21 -01:00",
|
||||
"tags" : [
|
||||
"ex",
|
||||
"do",
|
||||
"occaecat",
|
||||
"reprehenderit",
|
||||
"anim",
|
||||
"laboris",
|
||||
"cillum"
|
||||
],
|
||||
"friends": [
|
||||
{
|
||||
"id" : 0,
|
||||
"name" : "Wendi Odonnell"
|
||||
},
|
||||
{
|
||||
"id" : 1,
|
||||
"name" : "Mayra Boyd"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Lee Gonzalez"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
- do:
|
||||
get:
|
||||
index: test
|
||||
type: test
|
||||
id: 1
|
||||
- length: { _source: 11 }
|
||||
- is_false: _source.friends.0.id
|
||||
- is_false: _source.friends.1.id
|
||||
- is_false: _source.friends.2.id
|
||||
- match: { _source.friends.0.name: "Wendi Odonnell" }
|
||||
- match: { _source.friends.1.name: "Mayra Boyd" }
|
||||
- match: { _source.friends.2.name: "Lee Gonzalez" }
|
||||
- match: { _source.tags.7: "new_value" }
|
||||
- length: { _source.address: 4 }
|
||||
- match: { _source.address.0: "713 Bartlett Place" }
|
||||
- match: { _source.address.1: "Accoville" }
|
||||
- match: { _source.address.2: "Puerto Rico" }
|
||||
- match: { _source.address.3: "9221" }
|
||||
- match: { _source.company: "atgen" }
|
||||
- match: { _source.gender: "MALE" }
|
||||
- match: { _source.eye_color: "brown" }
|
|
@ -80,6 +80,10 @@
|
|||
"type" : "boolean",
|
||||
"description" : "Specify whether query terms should be lowercased"
|
||||
},
|
||||
"pipeline": {
|
||||
"type" : "string",
|
||||
"description" : "Ingest pipeline to set on index requests made by this action. (default: none)"
|
||||
},
|
||||
"preference": {
|
||||
"type" : "string",
|
||||
"description" : "Specify the node or shard the operation should be performed on (default: random)"
|
||||
|
|
|
@ -1,6 +1,23 @@
|
|||
Test Suite:
|
||||
===========
|
||||
|
||||
[NOTE]
|
||||
.Required settings
|
||||
=======================================
|
||||
Certain tests require specific settings to be applied to the
|
||||
Elasticsearch instance in order to pass. You should run
|
||||
Elasticsearch as follows:
|
||||
|
||||
[source,sh]
|
||||
---------------------
|
||||
bin/elasticsearch --script.inline true --node.testattr test --path.repo /tmp --repositories.url.allowed_urls 'http://snapshot.*'
|
||||
---------------------
|
||||
|
||||
=======================================
|
||||
|
||||
Test file structure
|
||||
--------------------
|
||||
|
||||
A YAML test file consists of:
|
||||
* an optional `setup` section, followed by
|
||||
* one or more test sections
|
||||
|
|
Loading…
Reference in New Issue