honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/query-refactoring

This commit is contained in:
Christoph Büscher 2015-08-26 10:23:03 +02:00
parent 96b5cebfb1 e8834cc78c
commit efadf87371
18 changed files with 409 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.core.StringFieldMapper;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.analysis.IndicesAnalysisService;
@ -215,19 +216,38 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable
Map<String, NamedAnalyzer> analyzers = newHashMap();
for (AnalyzerProvider analyzerFactory : analyzerProviders.values()) {
/*
* Lucene defaults positionOffsetGap to 0 in all analyzers but
* Elasticsearch defaults them to 0 only before version 2.1
* and 100 afterwards so we override the positionOffsetGap if it
* doesn't match here.
*/
int overridePositionOffsetGap = StringFieldMapper.Defaults.positionOffsetGap(Version.indexCreated(indexSettings));
if (analyzerFactory instanceof CustomAnalyzerProvider) {
((CustomAnalyzerProvider) analyzerFactory).build(this);
/*
* Custom analyzers already default to the correct, version
* dependent positionOffsetGap and the user is be able to
* configure the positionOffsetGap directly on the analyzer so
* we disable overriding the positionOffsetGap to preserve the
* user's setting.
*/
overridePositionOffsetGap = Integer.MIN_VALUE;
}
Analyzer analyzerF = analyzerFactory.get();
if (analyzerF == null) {
throw new IllegalArgumentException("analyzer [" + analyzerFactory.name() + "] created null analyzer");
}
NamedAnalyzer analyzer;
// if we got a named analyzer back, use it...
if (analyzerF instanceof NamedAnalyzer) {
// if we got a named analyzer back, use it...
analyzer = (NamedAnalyzer) analyzerF;
if (overridePositionOffsetGap >= 0 && analyzer.getPositionIncrementGap(analyzer.name()) != overridePositionOffsetGap) {
// unless the positionOffsetGap needs to be overridden
analyzer = new NamedAnalyzer(analyzer, overridePositionOffsetGap);
}
} else {
analyzer = new NamedAnalyzer(analyzerFactory.name(), analyzerFactory.scope(), analyzerF);
analyzer = new NamedAnalyzer(analyzerFactory.name(), analyzerFactory.scope(), analyzerF, overridePositionOffsetGap);
}
analyzers.put(analyzerFactory.name(), analyzer);
analyzers.put(Strings.toCamelCase(analyzerFactory.name()), analyzer);

5
core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java
View File

@ -19,10 +19,12 @@
package org.elasticsearch.index.analysis;
import org.elasticsearch.Version;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.core.StringFieldMapper;
import org.elasticsearch.index.settings.IndexSettings;
import java.util.List;
@ -77,7 +79,8 @@ public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider<Custom
tokenFilters.add(tokenFilter);
}
int positionOffsetGap = analyzerSettings.getAsInt("position_offset_gap", 0);
int positionOffsetGapDefault = StringFieldMapper.Defaults.positionOffsetGap(Version.indexCreated(indexSettings));
int positionOffsetGap = analyzerSettings.getAsInt("position_offset_gap", positionOffsetGapDefault);
int offsetGap = analyzerSettings.getAsInt("offset_gap", -1);
this.customAnalyzer = new CustomAnalyzer(tokenizer,

52
core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View File

@ -100,33 +100,36 @@ class DocumentParser implements Closeable {
context.reset(parser, new ParseContext.Document(), source);
// will result in START_OBJECT
int countDownTokens = 0;
XContentParser.Token token = parser.nextToken();
if (token != XContentParser.Token.START_OBJECT) {
throw new MapperParsingException("Malformed content, must start with an object");
}
boolean emptyDoc = false;
token = parser.nextToken();
if (token == XContentParser.Token.END_OBJECT) {
// empty doc, we can handle it...
emptyDoc = true;
} else if (token != XContentParser.Token.FIELD_NAME) {
throw new MapperParsingException("Malformed content, after first object, either the type field or the actual properties should exist");
}
for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
metadataMapper.preParse(context);
}
if (!emptyDoc) {
Mapper update = parseObject(context, mapping.root);
if (update != null) {
context.addDynamicMappingsUpdate(update);
if (mapping.root.isEnabled()) {
boolean emptyDoc = false;
token = parser.nextToken();
if (token == XContentParser.Token.END_OBJECT) {
// empty doc, we can handle it...
emptyDoc = true;
} else if (token != XContentParser.Token.FIELD_NAME) {
throw new MapperParsingException("Malformed content, after first object, either the type field or the actual properties should exist");
}
}
for (int i = 0; i < countDownTokens; i++) {
parser.nextToken();
for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
metadataMapper.preParse(context);
}
if (emptyDoc == false) {
Mapper update = parseObject(context, mapping.root);
if (update != null) {
context.addDynamicMappingsUpdate(update);
}
}
for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
metadataMapper.postParse(context);
}
} else {
// entire type is disabled
parser.skipChildren();
}
// try to parse the next token, this should be null if the object is ended properly
@ -135,12 +138,11 @@ class DocumentParser implements Closeable {
&& source.parser() == null && parser != null) {
// only check for end of tokens if we created the parser here
token = parser.nextToken();
assert token == null; // double check, in tests, that we didn't end parsing early
if (token != null) {
throw new IllegalArgumentException("Malformed content, found extra data after parsing: " + token);
}
}
for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
metadataMapper.postParse(context);
}
} catch (Throwable e) {
// if its already a mapper parsing exception, no need to wrap it...
if (e instanceof MapperParsingException) {

37
core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -52,6 +53,7 @@ import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
public class StringFieldMapper extends FieldMapper implements AllFieldMapper.IncludeInAll {
public static final String CONTENT_TYPE = "string";
private static final int POSITION_OFFSET_GAP_USE_ANALYZER = -1;
public static class Defaults {
public static final MappedFieldType FIELD_TYPE = new StringFieldType();
@ -62,15 +64,36 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
// NOTE, when adding defaults here, make sure you add them in the builder
public static final String NULL_VALUE = null;
public static final int POSITION_OFFSET_GAP = 0;
/**
* Post 2.0 default for position_offset_gap. Set to 100 so that
* phrase queries of reasonably high slop will not match across field
* values.
*/
public static final int POSITION_OFFSET_GAP = 100;
public static final int POSITION_OFFSET_GAP_PRE_2_0 = 0;
public static final int IGNORE_ABOVE = -1;
/**
* The default position_offset_gap for a particular version of Elasticsearch.
*/
public static int positionOffsetGap(Version version) {
if (version.before(Version.V_2_0_0_beta1)) {
return POSITION_OFFSET_GAP_PRE_2_0;
}
return POSITION_OFFSET_GAP;
}
}
public static class Builder extends FieldMapper.Builder<Builder, StringFieldMapper> {
protected String nullValue = Defaults.NULL_VALUE;
protected int positionOffsetGap = Defaults.POSITION_OFFSET_GAP;
/**
* The distance between tokens from different values in the same field.
* POSITION_OFFSET_GAP_USE_ANALYZER means default to the analyzer's
* setting which in turn defaults to Defaults.POSITION_OFFSET_GAP.
*/
protected int positionOffsetGap = POSITION_OFFSET_GAP_USE_ANALYZER;
protected int ignoreAbove = Defaults.IGNORE_ABOVE;
@ -102,7 +125,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
@Override
public StringFieldMapper build(BuilderContext context) {
if (positionOffsetGap > 0) {
if (positionOffsetGap != POSITION_OFFSET_GAP_USE_ANALYZER) {
fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionOffsetGap));
fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionOffsetGap));
fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionOffsetGap));
@ -154,7 +177,11 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
builder.searchQuotedAnalyzer(analyzer);
iterator.remove();
} else if (propName.equals("position_offset_gap")) {
builder.positionOffsetGap(XContentMapValues.nodeIntegerValue(propNode, -1));
int newPositionOffsetGap = XContentMapValues.nodeIntegerValue(propNode, -1);
if (newPositionOffsetGap < 0) {
throw new MapperParsingException("positions_offset_gap less than 0 aren't allowed.");
}
builder.positionOffsetGap(newPositionOffsetGap);
// we need to update to actual analyzers if they are not set in this case...
// so we can inject the position offset gap...
if (builder.fieldType().indexAnalyzer() == null) {
@ -354,7 +381,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
builder.field("include_in_all", false);
}
if (includeDefaults || positionOffsetGap != Defaults.POSITION_OFFSET_GAP) {
if (includeDefaults || positionOffsetGap != POSITION_OFFSET_GAP_USE_ANALYZER) {
builder.field("position_offset_gap", positionOffsetGap);
}
NamedAnalyzer searchQuoteAnalyzer = fieldType().searchQuoteAnalyzer();

11
core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.bwcompat;
import com.google.common.base.Predicate;
import com.google.common.util.concurrent.ListenableFuture;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -40,6 +41,7 @@ import org.elasticsearch.common.util.MultiDataPathUpgrader;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.mapper.string.StringFieldMapperPositionOffsetGapTests;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.shard.MergePolicyConfig;
import org.elasticsearch.indices.recovery.RecoverySettings;
@ -330,6 +332,7 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
assertNewReplicasWork(indexName);
assertUpgradeWorks(indexName, isLatestLuceneVersion(version));
assertDeleteByQueryWorked(indexName, version);
assertPositionOffsetGapDefaults(indexName, version);
unloadIndex(indexName);
}
@ -442,6 +445,14 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
assertEquals(0, searchReq.get().getHits().getTotalHits());
}
void assertPositionOffsetGapDefaults(String indexName, Version version) throws Exception {
if (version.before(Version.V_2_0_0_beta1)) {
StringFieldMapperPositionOffsetGapTests.assertGapIsZero(client(), indexName, "doc");
} else {
StringFieldMapperPositionOffsetGapTests.assertGapIsOneHundred(client(), indexName, "doc");
}
}
void assertUpgradeWorks(String indexName, boolean alreadyLatest) throws Exception {
if (alreadyLatest == false) {
UpgradeIT.assertNotUpgraded(client(), indexName);

64
core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.json.JsonXContentParser;
import org.elasticsearch.test.ESSingleNodeTestCase;
// TODO: make this a real unit test
public class DocumentParserTests extends ESSingleNodeTestCase {
public void testTypeDisabled() throws Exception {
DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.field("enabled", false).endObject().endObject().string();
DocumentMapper mapper = mapperParser.parse(mapping);
BytesReference bytes = XContentFactory.jsonBuilder()
.startObject()
.field("field", "1234")
.endObject().bytes();
ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
assertNull(doc.rootDoc().getField("field"));
}
public void testFieldDisabled() throws Exception {
DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("foo").field("enabled", false).endObject()
.startObject("bar").field("type", "integer").endObject()
.endObject().endObject().endObject().string();
DocumentMapper mapper = mapperParser.parse(mapping);
BytesReference bytes = XContentFactory.jsonBuilder()
.startObject()
.field("foo", "1234")
.field("bar", 10)
.endObject().bytes();
ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
assertNull(doc.rootDoc().getField("foo"));
assertNotNull(doc.rootDoc().getField("bar"));
}
}

158
core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionOffsetGapTests.java Normal file
View File

@ -0,0 +1,158 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.string;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.test.ESSingleNodeTestCase;
import java.io.IOException;
import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.containsString;
/**
* Tests that position_offset_gap is read from the mapper and applies as
* expected in queries.
*/
public class StringFieldMapperPositionOffsetGapTests extends ESSingleNodeTestCase {
/**
* The default position_offset_gap should be large enough that most
* "sensible" queries phrase slops won't match across values.
*/
public void testDefault() throws IOException {
assertGapIsOneHundred(client(), "test", "test");
}
/**
* Asserts that the post-2.0 default is being applied.
*/
public static void assertGapIsOneHundred(Client client, String indexName, String type) throws IOException {
testGap(client(), indexName, type, 100);
// No match across gap using default slop with default positionOffsetGap
assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two")).get(), 0);
// Nor with small-ish values
assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(5)).get(), 0);
assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(50)).get(), 0);
// But huge-ish values still match
assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(500)).get(), 1);
}
public void testZero() throws IOException {
setupGapInMapping(0);
assertGapIsZero(client(), "test", "test");
}
/**
* Asserts that the pre-2.0 default has been applied or explicitly
* configured.
*/
public static void assertGapIsZero(Client client, String indexName, String type) throws IOException {
testGap(client, indexName, type, 0);
/*
* Phrases match across different values using default slop with pre-2.0 default
* position_offset_gap.
*/
assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two")).get(), 1);
}
public void testLargerThanDefault() throws IOException {
setupGapInMapping(10000);
testGap(client(), "test", "test", 10000);
}
public void testSmallerThanDefault() throws IOException {
setupGapInMapping(2);
testGap(client(), "test", "test", 2);
}
public void testNegativeIsError() throws IOException {
try {
setupGapInMapping(-1);
fail("Expected an error");
} catch (MapperParsingException e) {
assertThat(ExceptionsHelper.detailedMessage(e), containsString("positions_offset_gap less than 0 aren't allowed"));
}
}
/**
* Tests that the default actually defaults to the position_offset_gap
* configured in the analyzer. This behavior is very old and a little
* strange but not worth breaking some thought.
*/
public void testDefaultDefaultsToAnalyzer() throws IOException {
XContentBuilder settings = XContentFactory.jsonBuilder().startObject().startObject("analysis").startObject("analyzer")
.startObject("gappy");
settings.field("type", "custom");
settings.field("tokenizer", "standard");
settings.field("position_offset_gap", 2);
setupAnalyzer(settings, "gappy");
testGap(client(), "test", "test", 2);
}
/**
* Build an index named "test" with a field named "string" with the provided
* positionOffsetGap that uses the standard analyzer.
*/
private void setupGapInMapping(int positionOffsetGap) throws IOException {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("properties").startObject("string");
mapping.field("type", "string");
mapping.field("position_offset_gap", positionOffsetGap);
client().admin().indices().prepareCreate("test").addMapping("test", mapping).get();
}
/**
* Build an index named "test" with the provided settings and and a field
* named "string" that uses the specified analyzer and default
* position_offset_gap.
*/
private void setupAnalyzer(XContentBuilder settings, String analyzer) throws IOException {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("properties").startObject("string");
mapping.field("type", "string");
mapping.field("analyzer", analyzer);
client().admin().indices().prepareCreate("test").addMapping("test", mapping).setSettings(settings).get();
}
private static void testGap(Client client, String indexName, String type, int positionOffsetGap) throws IOException {
client.prepareIndex(indexName, type, "position_gap_test").setSource("string", ImmutableList.of("one", "two three")).setRefresh(true).get();
// Baseline - phrase query finds matches in the same field value
assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "two three")).get(), 1);
if (positionOffsetGap > 0) {
// No match across gaps when slop < position gap
assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(positionOffsetGap - 1)).get(),
0);
}
// Match across gaps when slop >= position gap
assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(positionOffsetGap)).get(), 1);
assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(positionOffsetGap + 1)).get(), 1);
}
}

5
dev-tools/es_release_notes.pl
View File

@ -29,7 +29,7 @@ my $Issue_URL = "http://github.com/${User_Repo}issues/";
my @Groups = qw(
breaking deprecation feature
enhancement bug regression build doc test
enhancement bug regression upgrade build doc test
);
my %Group_Labels = (
breaking => 'Breaking changes',
@ -39,8 +39,9 @@ my %Group_Labels = (
feature => 'New features',
enhancement => 'Enhancements',
bug => 'Bug fixes',
regression => 'Regression',
regression => 'Regressions',
test => 'Tests',
upgrade => 'Upgrades',
other => 'NOT CLASSIFIED',
);

5
distribution/src/main/resources/bin/elasticsearch
View File

@ -121,7 +121,10 @@ case `uname` in
;;
esac
export HOSTNAME=`hostname -s`
# full hostname passed through cut for portability on systems that do not support hostname -s
# export on separate line for shells that do not support combining definition and export
HOSTNAME=`hostname | cut -d. -f1`
export HOSTNAME
# manual parsing to find out, if process should be detached
daemonized=`echo $* | grep -E -- '(^-d |-d$| -d |--daemonize$|--daemonize )'`

5
distribution/src/main/resources/bin/plugin
View File

@ -103,6 +103,9 @@ if [ -e "$CONF_FILE" ]; then
esac
fi
export HOSTNAME=`hostname -s`
# full hostname passed through cut for portability on systems that do not support hostname -s
# export on separate line for shells that do not support combining definition and export
HOSTNAME=`hostname | cut -d. -f1`
export HOSTNAME
eval "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS -Xmx64m -Xms16m -Delasticsearch -Des.path.home="\"$ES_HOME\"" $properties -cp "\"$ES_HOME/lib/*\"" org.elasticsearch.plugins.PluginManagerCliParser $args

8
docs/reference/analysis/analyzers/custom-analyzer.asciidoc
View File

@ -20,8 +20,10 @@ filters.
|`char_filter` |An optional list of logical / registered name of char
filters.
|`position_offset_gap` |An optional number of positions to increment
between each field value of a field using this analyzer.
|`position_offset_gap` |An optional number of positions to increment
between each field value of a field using this analyzer. Defaults to 100.
100 was chosen because it prevents phrase queries with reasonably large
slops (less than 100) from matching terms across field values.
|=======================================================================
Here is an example:
@ -30,7 +32,7 @@ Here is an example:
--------------------------------------------------
index :
analysis :
analyzer :
analyzer :
myAnalyzer2 :
type : custom
tokenizer : myTokenizer1

18
docs/reference/docs/index_.asciidoc
View File

@ -353,7 +353,23 @@ occurs, so that the document appears in search results immediately, the
`refresh` parameter can be set to `true`. Setting this option to `true` should
*ONLY* be done after careful thought and verification that it does not lead to
poor performance, both from an indexing and a search standpoint. Note, getting
a document using the get API is completely realtime.
a document using the get API is completely realtime and doesn't require a
refresh.
[float]
[[index-noop]]
=== Noop Updates
When updating a document using the index api a new version of the document is
always created even if the document hasn't changed. If this isn't acceptable
use the `_update` api with `detect_noop` set to true. This option isn't
available on the index api because the index api doesn't fetch the old source
and isn't able to compare it against the new source.
There isn't a hard and fast rule about when noop updates aren't acceptable.
It's a combination of lots of factors like how frequently your data source
sends updates that are actually noops and how many queries per second
elasticsearch runs on the shard with receiving the updates.
[float]
[[timeout]]

7
docs/reference/mapping/types/string.asciidoc
View File

@ -145,6 +145,11 @@ Defaults depend on the <<mapping-index,`index`>> setting:
The number of fake term positions which should be inserted between
each element of an array of strings. Defaults to 0.
The number of fake term position which should be inserted between each
element of an array of strings. Defaults to the position_offset_gap
configured on the analyzer which defaults to 100. 100 was chosen because it
prevents phrase queries with reasonably large slops (less than 100) from
matching terms across field values.
<<mapping-store,`store`>>::
@ -166,5 +171,3 @@ Defaults depend on the <<mapping-index,`index`>> setting:
Whether term vectors should be stored for an <<mapping-index,`analyzed`>>
field. Defaults to `no`.

11
docs/reference/migration/migrate_2_0.asciidoc
View File

@ -13,13 +13,6 @@ latest 1.x version of Elasticsearch first, in order to upgrade your indices or
to delete the old indices. Elasticsearch will not start in the presence of old
indices.
[float]
=== Network binds to localhost only
Elasticsearch now binds to the loopback interface by default (usually
`127.0.0.1` or `::1`). The `network.host` setting can be specified to change
this behavior.
[float]
=== Elasticsearch migration plugin
@ -29,6 +22,8 @@ Elasticsearch 2.0. Please install and run the plugin *before* upgrading.
include::migrate_2_0/removals.asciidoc[]
include::migrate_2_0/network.asciidoc[]
include::migrate_2_0/striping.asciidoc[]
include::migrate_2_0/mapping.asciidoc[]
@ -55,4 +50,4 @@ include::migrate_2_0/settings.asciidoc[]
include::migrate_2_0/stats.asciidoc[]
include::migrate_2_0/java.asciidoc[]
include::migrate_2_0/java.asciidoc[]

11
docs/reference/migration/migrate_2_0/mapping.asciidoc
View File

@ -384,7 +384,10 @@ The `compress` and `compress_threshold` options have been removed from the
default. If you would like to increase compression levels, use the new
<<index-codec,`index.codec: best_compression`>> setting instead.
==== position_offset_gap
The default `position_offset_gap` is now 100. Indexes created in Elasticsearch
2.0.0 will default to using 100 and indexes created before that will continue
to use the old default of 0. This was done to prevent phrase queries from
matching across different values of the same term unexpectedly. Specifically,
100 was chosen to cause phrase queries with slops up to 99 to match only within
a single value of a field.

42
docs/reference/migration/migrate_2_0/network.asciidoc Normal file
View File

@ -0,0 +1,42 @@
=== Network changes
==== Bind to localhost
Elasticsearch 2.x will only bind to localhost by default. It will try to bind
to both 127.0.0.1 (IPv4) and [::1] (IPv6), but will work happily in
environments where only IPv4 or IPv6 is available. This change prevents
Elasticsearch from trying to connect to other nodes on your network unless you
specifically tell it to do so. When moving to production you should configure
the `network.host` parameter, either in the `elasticsearch.yml` config file or
on the command line:
[source,sh]
--------------------
bin/elasticsearch --network.host 192.168.1.5
bin/elasticsearch --network.host _non_loopback_
--------------------
The full list of options that network.host accepts can be found in the <<modules-network>>.
==== Multicast removed
Multicast has been removed (although it is still
{plugins}/discovery-multicast.html[provided as a plugin] for now). Instead,
and only when bound to localhost, Elasticsearch will use unicast to contact
the first 5 ports in the `transport.tcp.port` range, which defaults to
`9300-9400`.
This preserves the zero-config auto-clustering experience for the developer,
but it means that you will have to provide a list of <<unicast,unicast hosts>>
when moving to production, for instance:
[source,yaml]
---------------------
discovery.zen.ping.unicast.hosts: [ 192.168.1.2, 192.168.1.3 ]
---------------------
You don’t need to list all of the nodes in your cluster as unicast hosts, but
you should specify at least a quorum (majority) of master-eligible nodes. A
big cluster will typically have three dedicated master nodes, in which case we
recommend listing all three of them as unicast hosts.

6
docs/river/index.asciidoc
View File

@ -1,6 +0,0 @@
[[river]]
= Rivers
Rivers were deprecated in Elasticsearch 1.5 and removed in Elasticsearch 2.0.
See https://www.elastic.co/blog/deprecating_rivers for more details.

2
rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
View File

@ -1,6 +1,6 @@
{
"indices.shard_stores": {
"documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-shard-stores.html",
"documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-shards-stores.html",
"methods": ["GET"],
"url": {
"path": "/_shard_stores",