honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-23 05:15:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/client_aggs_parsing

Browse Source
This commit is contained in:
javanna 2017-05-19 18:17:06 +02:00 committed by Luca Cavanna
commit db0490343e
99 changed files with 883 additions and 1505 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/version.properties
View File

@ -1,6 +1,6 @@
# When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
elasticsearch = 6.0.0-alpha2
lucene = 7.0.0-snapshot-89f6d17
lucene = 7.0.0-snapshot-a0aef2f
# optional dependencies
spatial4j = 0.6

38
client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
View File

@ -55,27 +55,29 @@ public class PreBuiltTransportClient extends TransportClient {
}
/**
* Netty wants to do some unsafe things like use unsafe and replace a private field. This method disables these things by default, but
* can be overridden by setting the corresponding system properties.
* Netty wants to do some unwelcome things like use unsafe and replace a private field, or use a poorly considered buffer recycler. This
* method disables these things by default, but can be overridden by setting the corresponding system properties.
*/
@SuppressForbidden(reason = "set system properties to configure Netty")
private static void initializeNetty() {
final String noUnsafeKey = "io.netty.noUnsafe";
final String noUnsafe = System.getProperty(noUnsafeKey);
if (noUnsafe == null) {
// disable Netty from using unsafe
// while permissions are needed to set this, if a security exception is thrown the permission needed can either be granted or
// the system property can be set directly before starting the JVM; therefore, we do not catch a security exception here
System.setProperty(noUnsafeKey, Boolean.toString(true));
}
/*
* We disable three pieces of Netty functionality here:
* - we disable Netty from being unsafe
* - we disable Netty from replacing the selector key set
* - we disable Netty from using the recycler
*
* While permissions are needed to read and set these, the permissions needed here are innocuous and thus should simply be granted
* rather than us handling a security exception here.
*/
setSystemPropertyIfUnset("io.netty.noUnsafe", Boolean.toString(true));
setSystemPropertyIfUnset("io.netty.noKeySetOptimization", Boolean.toString(true));
setSystemPropertyIfUnset("io.netty.recycler.maxCapacityPerThread", Integer.toString(0));
}
final String noKeySetOptimizationKey = "io.netty.noKeySetOptimization";
final String noKeySetOptimization = System.getProperty(noKeySetOptimizationKey);
if (noKeySetOptimization == null) {
// disable Netty from replacing the selector key set
// while permissions are needed to set this, if a security exception is thrown the permission needed can either be granted or
// the system property can be set directly before starting the JVM; therefore, we do not catch a security exception here
System.setProperty(noKeySetOptimizationKey, Boolean.toString(true));
@SuppressForbidden(reason = "set system properties to configure Netty")
private static void setSystemPropertyIfUnset(final String key, final String value) {
final String currentValue = System.getProperty(key);
if (currentValue == null) {
System.setProperty(key, value);
}
}

1
core/licenses/lucene-analyzers-common-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
e69234c2e898d86a53edbe8d22e33bebc45286cd

1
core/licenses/lucene-analyzers-common-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
5e191674c50c9d99c9838da52cbf67c411998f4e

1
core/licenses/lucene-backward-codecs-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
48172a8e1fe6562f55ab671d42af53652794d5df

1
core/licenses/lucene-backward-codecs-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
45bc34ab640d5d1a7491b523631b902f20db5384

1
core/licenses/lucene-core-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
3dab251d4c7ab4ff5095e5f1d1e127ec2cf3c07d

1
core/licenses/lucene-core-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
b44d86e9077443c3ba4918a85603734461c6b448

1
core/licenses/lucene-grouping-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
c01ae8a23b733d75d058a76bd85fcb49b9fd06fd

1
core/licenses/lucene-grouping-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
409b616d40e2041a02890b2dc477ed845e3121e9

1
core/licenses/lucene-highlighter-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
c53df048b97946fe66035505306b5651b702adb1

1
core/licenses/lucene-highlighter-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
cfac105541315e2ca54955f681b410a7aa3bbb9d

1
core/licenses/lucene-join-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
1ecb349ba29abab75359e5125ac8a94fc81441d5

1
core/licenses/lucene-join-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
993c1331130dd26c632b964fd8caac259bb9f3fc

1
core/licenses/lucene-memory-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
e5f53b38652b1284ff254fba39e624ec117aef7d

1
core/licenses/lucene-memory-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
ec1460a28850410112a6349a7fff27df31242295

1
core/licenses/lucene-misc-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
2f340ed3f46d6b4c89fa31975b675c19028c15eb

1
core/licenses/lucene-misc-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
57d342dbe68cf05361ccfda6bb76f2410cac900b

1
core/licenses/lucene-queries-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
a13862fb62cc1e516d16d6b6bb3cdb906c4925f6

1
core/licenses/lucene-queries-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
5ed10847b6a2353ac66decd5a2ee1a1d34353049

1
core/licenses/lucene-queryparser-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
4e014f72a588453bae7dd1a555d741cf3bf39032

1
core/licenses/lucene-queryparser-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
23ce6c2ea59287d8fe4fe31f466e9a58a1efe7b5

1
core/licenses/lucene-sandbox-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
5e87d61c604d6b1c0ee5c38f09441d1b8b9c8c2b

1
core/licenses/lucene-sandbox-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
78bda71c8e65428927136f81112a031aa9cd04d4

1
core/licenses/lucene-spatial-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
be14aa163b339403d8ec904493c1be5dfa9baeaf

1
core/licenses/lucene-spatial-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
1e7ea95e6197176015b13551c7496be4867ede45

1
core/licenses/lucene-spatial-extras-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
a2c13be0fe4c5a98a30ec6ae673be1442409817c

1
core/licenses/lucene-spatial-extras-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
5ae4ecd6c478456395ae9a3f954b8afc13629bb9

1
core/licenses/lucene-spatial3d-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
92b8282e474845fdae31f9f239f953bc7164401f

1
core/licenses/lucene-spatial3d-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d5d1a81fc290b9660a49557f848dc2a3c4f2048b

1
core/licenses/lucene-suggest-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
1c4aaea267ed41657ebf01769bfddbcab5b27414

1
core/licenses/lucene-suggest-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d77cdd8f2782062a3b4c319c64f0fa4d804aafed

38
core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
View File

@ -74,14 +74,15 @@ public final class AnalysisRegistry implements Closeable {
Map<String, AnalysisProvider<TokenizerFactory>> tokenizers,
Map<String, AnalysisProvider<AnalyzerProvider<?>>> analyzers,
Map<String, AnalysisProvider<AnalyzerProvider<?>>> normalizers,
Map<String, PreConfiguredTokenFilter> preConfiguredTokenFilters) {
Map<String, PreConfiguredTokenFilter> preConfiguredTokenFilters,
Map<String, PreConfiguredTokenizer> preConfiguredTokenizers) {
this.environment = environment;
this.charFilters = unmodifiableMap(charFilters);
this.tokenFilters = unmodifiableMap(tokenFilters);
this.tokenizers = unmodifiableMap(tokenizers);
this.analyzers = unmodifiableMap(analyzers);
this.normalizers = unmodifiableMap(normalizers);
prebuiltAnalysis = new PrebuiltAnalysis(preConfiguredTokenFilters);
prebuiltAnalysis = new PrebuiltAnalysis(preConfiguredTokenFilters, preConfiguredTokenizers);
}
/**
@ -169,12 +170,12 @@ public final class AnalysisRegistry implements Closeable {
*/
tokenFilters.put("synonym", requiresAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings)));
tokenFilters.put("synonym_graph", requiresAnalysisSettings((is, env, name, settings) -> new SynonymGraphTokenFilterFactory(is, env, this, name, settings)));
return buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.tokenFilterFactories);
return buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.preConfiguredTokenFilters);
}
public Map<String, TokenizerFactory> buildTokenizerFactories(IndexSettings indexSettings) throws IOException {
final Map<String, Settings> tokenizersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_TOKENIZER);
return buildMapping(Component.TOKENIZER, indexSettings, tokenizersSettings, tokenizers, prebuiltAnalysis.tokenizerFactories);
return buildMapping(Component.TOKENIZER, indexSettings, tokenizersSettings, tokenizers, prebuiltAnalysis.preConfiguredTokenizers);
}
public Map<String, CharFilterFactory> buildCharFilterFactories(IndexSettings indexSettings) throws IOException {
@ -394,31 +395,22 @@ public final class AnalysisRegistry implements Closeable {
private static class PrebuiltAnalysis implements Closeable {
final Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<?>>> analyzerProviderFactories;
final Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizerFactories;
final Map<String, ? extends AnalysisProvider<TokenFilterFactory>> tokenFilterFactories;
final Map<String, ? extends AnalysisProvider<TokenFilterFactory>> preConfiguredTokenFilters;
final Map<String, ? extends AnalysisProvider<TokenizerFactory>> preConfiguredTokenizers;
final Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> charFilterFactories;
private PrebuiltAnalysis(Map<String, PreConfiguredTokenFilter> preConfiguredTokenFilters) {
private PrebuiltAnalysis(
Map<String, PreConfiguredTokenFilter> preConfiguredTokenFilters,
Map<String, PreConfiguredTokenizer> preConfiguredTokenizers) {
Map<String, PreBuiltAnalyzerProviderFactory> analyzerProviderFactories = new HashMap<>();
Map<String, PreBuiltTokenizerFactoryFactory> tokenizerFactories = new HashMap<>();
Map<String, PreBuiltCharFilterFactoryFactory> charFilterFactories = new HashMap<>();
// Analyzers
for (PreBuiltAnalyzers preBuiltAnalyzerEnum : PreBuiltAnalyzers.values()) {
String name = preBuiltAnalyzerEnum.name().toLowerCase(Locale.ROOT);
analyzerProviderFactories.put(name, new PreBuiltAnalyzerProviderFactory(name, AnalyzerScope.INDICES, preBuiltAnalyzerEnum.getAnalyzer(Version.CURRENT)));
}
// Tokenizers
for (PreBuiltTokenizers preBuiltTokenizer : PreBuiltTokenizers.values()) {
String name = preBuiltTokenizer.name().toLowerCase(Locale.ROOT);
tokenizerFactories.put(name, new PreBuiltTokenizerFactoryFactory(preBuiltTokenizer.getTokenizerFactory(Version.CURRENT)));
}
// Tokenizer aliases
tokenizerFactories.put("nGram", new PreBuiltTokenizerFactoryFactory(PreBuiltTokenizers.NGRAM.getTokenizerFactory(Version.CURRENT)));
tokenizerFactories.put("edgeNGram", new PreBuiltTokenizerFactoryFactory(PreBuiltTokenizers.EDGE_NGRAM.getTokenizerFactory(Version.CURRENT)));
tokenizerFactories.put("PathHierarchy", new PreBuiltTokenizerFactoryFactory(PreBuiltTokenizers.PATH_HIERARCHY.getTokenizerFactory(Version.CURRENT)));
// Char Filters
for (PreBuiltCharFilters preBuiltCharFilter : PreBuiltCharFilters.values()) {
String name = preBuiltCharFilter.name().toLowerCase(Locale.ROOT);
@ -429,8 +421,8 @@ public final class AnalysisRegistry implements Closeable {
this.analyzerProviderFactories = Collections.unmodifiableMap(analyzerProviderFactories);
this.charFilterFactories = Collections.unmodifiableMap(charFilterFactories);
this.tokenizerFactories = Collections.unmodifiableMap(tokenizerFactories);
tokenFilterFactories = preConfiguredTokenFilters;
this.preConfiguredTokenFilters = preConfiguredTokenFilters;
this.preConfiguredTokenizers = preConfiguredTokenizers;
}
public AnalysisModule.AnalysisProvider<CharFilterFactory> getCharFilterFactory(String name) {
@ -438,11 +430,11 @@ public final class AnalysisRegistry implements Closeable {
}
public AnalysisModule.AnalysisProvider<TokenFilterFactory> getTokenFilterFactory(String name) {
return tokenFilterFactories.get(name);
return preConfiguredTokenFilters.get(name);
}
public AnalysisModule.AnalysisProvider<TokenizerFactory> getTokenizerFactory(String name) {
return tokenizerFactories.get(name);
return preConfiguredTokenizers.get(name);
}
public AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> getAnalyzerProvider(String name) {

50
core/src/main/java/org/elasticsearch/index/analysis/PreBuiltTokenizerFactoryFactory.java
View File

@ -1,50 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.analysis;
import org.elasticsearch.Version;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.indices.analysis.PreBuiltTokenizers;
import java.io.IOException;
public class PreBuiltTokenizerFactoryFactory implements AnalysisModule.AnalysisProvider<TokenizerFactory> {
private final TokenizerFactory tokenizerFactory;
public PreBuiltTokenizerFactoryFactory(TokenizerFactory tokenizerFactory) {
this.tokenizerFactory = tokenizerFactory;
}
public TokenizerFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
Version indexVersion = Version.indexCreated(settings);
if (!Version.CURRENT.equals(indexVersion)) {
PreBuiltTokenizers preBuiltTokenizers = PreBuiltTokenizers.getOrDefault(name, null);
if (preBuiltTokenizers != null) {
return preBuiltTokenizers.getTokenizerFactory(indexVersion);
}
}
return tokenizerFactory;
}
}

64
core/src/main/java/org/elasticsearch/index/analysis/PreConfiguredAnalysisComponent.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.analysis;
import org.elasticsearch.Version;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.indices.analysis.PreBuiltCacheFactory;
import java.io.IOException;
/**
* Shared implementation for pre-configured analysis components.
*/
public abstract class PreConfiguredAnalysisComponent<T> implements AnalysisModule.AnalysisProvider<T> {
private final String name;
private final PreBuiltCacheFactory.PreBuiltCache<T> cache;
protected PreConfiguredAnalysisComponent(String name, PreBuiltCacheFactory.CachingStrategy cache) {
this.name = name;
this.cache = PreBuiltCacheFactory.getCache(cache);
}
@Override
public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
Version versionCreated = Version.indexCreated(settings);
synchronized (this) {
T factory = cache.get(versionCreated);
if (factory == null) {
factory = create(versionCreated);
cache.put(versionCreated, factory);
}
return factory;
}
}
/**
* The name of the analysis component in the API.
*/
public String getName() {
return name;
}
protected abstract T create(Version version);
}

87
core/src/main/java/org/elasticsearch/index/analysis/PreConfiguredTokenFilter.java
View File

@ -22,21 +22,16 @@ package org.elasticsearch.index.analysis;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.elasticsearch.Version;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.indices.analysis.PreBuiltCacheFactory;
import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy;
import java.io.IOException;
import java.util.function.BiFunction;
import java.util.function.Function;
/**
* Provides pre-configured, shared {@link TokenFilter}s.
*/
public final class PreConfiguredTokenFilter implements AnalysisModule.AnalysisProvider<TokenFilterFactory> {
public final class PreConfiguredTokenFilter extends PreConfiguredAnalysisComponent<TokenFilterFactory> {
/**
* Create a pre-configured token filter that may not vary at all.
*/
@ -60,35 +55,19 @@ public final class PreConfiguredTokenFilter implements AnalysisModule.AnalysisPr
*/
public static PreConfiguredTokenFilter elasticsearchVersion(String name, boolean useFilterForMultitermQueries,
BiFunction<TokenStream, org.elasticsearch.Version, TokenStream> create) {
return new PreConfiguredTokenFilter(name, useFilterForMultitermQueries, CachingStrategy.ELASTICSEARCH,
(tokenStream, version) -> create.apply(tokenStream, version));
return new PreConfiguredTokenFilter(name, useFilterForMultitermQueries, CachingStrategy.ELASTICSEARCH, create);
}
private final String name;
private final boolean useFilterForMultitermQueries;
private final PreBuiltCacheFactory.PreBuiltCache<TokenFilterFactory> cache;
private final BiFunction<TokenStream, Version, TokenStream> create;
private PreConfiguredTokenFilter(String name, boolean useFilterForMultitermQueries,
PreBuiltCacheFactory.CachingStrategy cache, BiFunction<TokenStream, Version, TokenStream> create) {
this.name = name;
super(name, cache);
this.useFilterForMultitermQueries = useFilterForMultitermQueries;
this.cache = PreBuiltCacheFactory.getCache(cache);
this.create = create;
}
@Override
public TokenFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
return getTokenFilterFactory(Version.indexCreated(settings));
}
/**
* The name of the {@link TokenFilter} in the API.
*/
public String getName() {
return name;
}
/**
* Can this {@link TokenFilter} be used in multi-term queries?
*/
@ -98,42 +77,36 @@ public final class PreConfiguredTokenFilter implements AnalysisModule.AnalysisPr
private interface MultiTermAwareTokenFilterFactory extends TokenFilterFactory, MultiTermAwareComponent {}
private synchronized TokenFilterFactory getTokenFilterFactory(final Version version) {
TokenFilterFactory factory = cache.get(version);
if (factory == null) {
if (useFilterForMultitermQueries) {
factory = new MultiTermAwareTokenFilterFactory() {
@Override
public String name() {
return name;
}
@Override
protected TokenFilterFactory create(Version version) {
if (useFilterForMultitermQueries) {
return new MultiTermAwareTokenFilterFactory() {
@Override
public String name() {
return getName();
}
@Override
public TokenStream create(TokenStream tokenStream) {
return create.apply(tokenStream, version);
}
@Override
public TokenStream create(TokenStream tokenStream) {
return create.apply(tokenStream, version);
}
@Override
public Object getMultiTermComponent() {
return this;
}
};
} else {
factory = new TokenFilterFactory() {
@Override
public String name() {
return name;
}
@Override
public TokenStream create(TokenStream tokenStream) {
return create.apply(tokenStream, version);
}
};
}
cache.put(version, factory);
@Override
public Object getMultiTermComponent() {
return this;
}
};
}
return new TokenFilterFactory() {
@Override
public String name() {
return getName();
}
return factory;
@Override
public TokenStream create(TokenStream tokenStream) {
return create.apply(tokenStream, version);
}
};
}
}

128
core/src/main/java/org/elasticsearch/index/analysis/PreConfiguredTokenizer.java Normal file
View File

@ -0,0 +1,128 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.analysis;
import org.apache.lucene.analysis.Tokenizer;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.indices.analysis.PreBuiltCacheFactory;
import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy;
import java.util.function.Function;
import java.util.function.Supplier;
/**
* Provides pre-configured, shared {@link Tokenizer}s.
*/
public final class PreConfiguredTokenizer extends PreConfiguredAnalysisComponent<TokenizerFactory> {
/**
* Create a pre-configured tokenizer that may not vary at all.
*
* @param name the name of the tokenizer in the api
* @param create builds the tokenizer
* @param multiTermComponent null if this tokenizer shouldn't be used for multi-term queries, otherwise a supplier for the
* {@link TokenFilterFactory} that stands in for this tokenizer in multi-term queries.
*/
public static PreConfiguredTokenizer singleton(String name, Supplier<Tokenizer> create,
@Nullable Supplier<TokenFilterFactory> multiTermComponent) {
return new PreConfiguredTokenizer(name, CachingStrategy.ONE, version -> create.get(),
multiTermComponent == null ? null : version -> multiTermComponent.get());
}
/**
* Create a pre-configured tokenizer that may vary based on the Lucene version.
*
* @param name the name of the tokenizer in the api
* @param create builds the tokenizer
* @param multiTermComponent null if this tokenizer shouldn't be used for multi-term queries, otherwise a supplier for the
* {@link TokenFilterFactory} that stands in for this tokenizer in multi-term queries.
*/
public static PreConfiguredTokenizer luceneVersion(String name, Function<org.apache.lucene.util.Version, Tokenizer> create,
@Nullable Function<org.apache.lucene.util.Version, TokenFilterFactory> multiTermComponent) {
return new PreConfiguredTokenizer(name, CachingStrategy.LUCENE, version -> create.apply(version.luceneVersion),
multiTermComponent == null ? null : version -> multiTermComponent.apply(version.luceneVersion));
}
/**
* Create a pre-configured tokenizer that may vary based on the Elasticsearch version.
*
* @param name the name of the tokenizer in the api
* @param create builds the tokenizer
* @param multiTermComponent null if this tokenizer shouldn't be used for multi-term queries, otherwise a supplier for the
* {@link TokenFilterFactory} that stands in for this tokenizer in multi-term queries.
*/
public static PreConfiguredTokenizer elasticsearchVersion(String name, Function<org.elasticsearch.Version, Tokenizer> create,
@Nullable Function<Version, TokenFilterFactory> multiTermComponent) {
return new PreConfiguredTokenizer(name, CachingStrategy.ELASTICSEARCH, create, multiTermComponent);
}
private final Function<Version, Tokenizer> create;
private final Function<Version, TokenFilterFactory> multiTermComponent;
private PreConfiguredTokenizer(String name, PreBuiltCacheFactory.CachingStrategy cache, Function<Version, Tokenizer> create,
@Nullable Function<Version, TokenFilterFactory> multiTermComponent) {
super(name, cache);
this.create = create;
this.multiTermComponent = multiTermComponent;
}
/**
* Does this tokenizer has an equivalent component for analyzing multi-term queries?
*/
public boolean hasMultiTermComponent() {
return multiTermComponent != null;
}
private interface MultiTermAwareTokenizerFactory extends TokenizerFactory, MultiTermAwareComponent {}
@Override
protected TokenizerFactory create(Version version) {
if (multiTermComponent != null) {
return new MultiTermAwareTokenizerFactory() {
@Override
public String name() {
return getName();
}
@Override
public Tokenizer create() {
return create.apply(version);
}
@Override
public Object getMultiTermComponent() {
return multiTermComponent.apply(version);
}
};
} else {
return new TokenizerFactory() {
@Override
public String name() {
return getName();
}
@Override
public Tokenizer create() {
return create.apply(version);
}
};
}
}
}

43
core/src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java
View File

@ -1,43 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.SortedDocValues;
import java.util.Set;
/**
* Specialization of {@link AtomicFieldData} for parent/child mappings.
*/
public interface AtomicParentChildFieldData extends AtomicFieldData {
/**
* Return the set of types there is a mapping for.
*/
Set<String> types();
/**
* Return the mapping for the given type. The returned
* {@link SortedDocValues} will map doc IDs to the identifier of their
* parent.
*/
SortedDocValues getOrdinalsValues(String type);
}

7
core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
View File

@ -21,7 +21,7 @@ package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiDocValues;
/**
@ -42,4 +42,9 @@ public interface IndexOrdinalsFieldData extends IndexFieldData.Global<AtomicOrdi
@Override
IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception;
/**
* Returns the underlying {@link MultiDocValues.OrdinalMap} for this fielddata
* or null if global ordinals are not needed (constant value or single segment).
*/
MultiDocValues.OrdinalMap getOrdinalMap();
}

46
core/src/main/java/org/elasticsearch/index/fielddata/IndexParentChildFieldData.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
/**
* Soecialization of {@link IndexFieldData} for parent/child mappings.
*/
public interface IndexParentChildFieldData extends IndexFieldData.Global<AtomicParentChildFieldData> {
/**
* Load a global view of the ordinals for the given {@link IndexReader},
* potentially from a cache.
*/
@Override
IndexParentChildFieldData loadGlobal(DirectoryReader indexReader);
/**
* Load a global view of the ordinals for the given {@link IndexReader}.
*/
@Override
IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception;
}

6
core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
@ -99,6 +100,11 @@ public final class UidIndexFieldData implements IndexOrdinalsFieldData {
return new UidIndexFieldData(index, type, idFieldData.localGlobalDirect(indexReader));
}
@Override
public MultiDocValues.OrdinalMap getOrdinalMap() {
return idFieldData.getOrdinalMap();
}
static final class UidAtomicFieldData implements AtomicOrdinalsFieldData {
private final BytesRef prefix;

4
core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
View File

@ -74,7 +74,7 @@ public enum GlobalOrdinalsBuilder {
new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS)
);
}
return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(),
return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(),
atomicFD, ordinalMap, memorySizeInBytes, scriptFunction
);
}
@ -108,7 +108,7 @@ public enum GlobalOrdinalsBuilder {
subs[i] = atomicFD[i].getOrdinalsValues();
}
final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT);
return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(),
return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(),
atomicFD, ordinalMap, 0, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION
);
}

75
core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
View File

@ -20,6 +20,8 @@ package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.Accountable;
import org.elasticsearch.common.Nullable;
@ -29,23 +31,39 @@ import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.util.Collection;
import java.util.Collections;
import java.util.function.Function;
/**
* {@link IndexFieldData} base class for concrete global ordinals implementations.
*/
public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable {
public class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable {
private final String fieldName;
private final long memorySizeInBytes;
protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, long memorySizeInBytes) {
private final MultiDocValues.OrdinalMap ordinalMap;
private final Atomic[] atomicReaders;
private final Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction;
protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd,
MultiDocValues.OrdinalMap ordinalMap, long memorySizeInBytes, Function<SortedSetDocValues,
ScriptDocValues<?>> scriptFunction) {
super(indexSettings);
this.fieldName = fieldName;
this.memorySizeInBytes = memorySizeInBytes;
this.ordinalMap = ordinalMap;
this.atomicReaders = new Atomic[segmentAfd.length];
for (int i = 0; i < segmentAfd.length; i++) {
atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i);
}
this.scriptFunction = scriptFunction;
}
@Override
@ -88,4 +106,57 @@ public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponen
// TODO: break down ram usage?
return Collections.emptyList();
}
@Override
public AtomicOrdinalsFieldData load(LeafReaderContext context) {
return atomicReaders[context.ord];
}
@Override
public MultiDocValues.OrdinalMap getOrdinalMap() {
return ordinalMap;
}
private final class Atomic extends AbstractAtomicOrdinalsFieldData {
private final AtomicOrdinalsFieldData afd;
private final MultiDocValues.OrdinalMap ordinalMap;
private final int segmentIndex;
private Atomic(AtomicOrdinalsFieldData afd, MultiDocValues.OrdinalMap ordinalMap, int segmentIndex) {
super(scriptFunction);
this.afd = afd;
this.ordinalMap = ordinalMap;
this.segmentIndex = segmentIndex;
}
@Override
public SortedSetDocValues getOrdinalsValues() {
final SortedSetDocValues values = afd.getOrdinalsValues();
if (values.getValueCount() == ordinalMap.getValueCount()) {
// segment ordinals match global ordinals
return values;
}
final SortedSetDocValues[] bytesValues = new SortedSetDocValues[atomicReaders.length];
for (int i = 0; i < bytesValues.length; i++) {
bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues();
}
return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex);
}
@Override
public long ramBytesUsed() {
return afd.ramBytesUsed();
}
@Override
public Collection<Accountable> getChildResources() {
return afd.getChildResources();
}
@Override
public void close() {
}
}
}

99
core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java
View File

@ -1,99 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.Accountable;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
import java.util.Collection;
import java.util.function.Function;
/**
* {@link org.elasticsearch.index.fielddata.IndexFieldData} impl based on global ordinals.
*/
final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFieldData {
private final Atomic[] atomicReaders;
private final Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction;
InternalGlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd,
OrdinalMap ordinalMap, long memorySizeInBytes, Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) {
super(indexSettings, fieldName, memorySizeInBytes);
this.atomicReaders = new Atomic[segmentAfd.length];
for (int i = 0; i < segmentAfd.length; i++) {
atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i);
}
this.scriptFunction = scriptFunction;
}
@Override
public AtomicOrdinalsFieldData load(LeafReaderContext context) {
return atomicReaders[context.ord];
}
private final class Atomic extends AbstractAtomicOrdinalsFieldData {
private final AtomicOrdinalsFieldData afd;
private final OrdinalMap ordinalMap;
private final int segmentIndex;
private Atomic(AtomicOrdinalsFieldData afd, OrdinalMap ordinalMap, int segmentIndex) {
super(scriptFunction);
this.afd = afd;
this.ordinalMap = ordinalMap;
this.segmentIndex = segmentIndex;
}
@Override
public SortedSetDocValues getOrdinalsValues() {
final SortedSetDocValues values = afd.getOrdinalsValues();
if (values.getValueCount() == ordinalMap.getValueCount()) {
// segment ordinals match global ordinals
return values;
}
final SortedSetDocValues[] bytesValues = new SortedSetDocValues[atomicReaders.length];
for (int i = 0; i < bytesValues.length; i++) {
bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues();
}
return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex);
}
@Override
public long ramBytesUsed() {
return afd.ramBytesUsed();
}
@Override
public Collection<Accountable> getChildResources() {
return afd.getChildResources();
}
@Override
public void close() {
}
}
}

128
core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java
View File

@ -1,128 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.AtomicParentChildFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Set;
import static java.util.Collections.emptySet;
abstract class AbstractAtomicParentChildFieldData implements AtomicParentChildFieldData {
@Override
public final ScriptDocValues getScriptValues() {
return new ScriptDocValues.Strings(getBytesValues());
}
@Override
public final SortedBinaryDocValues getBytesValues() {
return new SortedBinaryDocValues() {
private final SortedDocValues[] perTypeValues;
private final BytesRef[] terms = new BytesRef[2];
private int count;
private int termsCursor;
{
Set<String> types = types();
perTypeValues = new SortedDocValues[types.size()];
int i = 0;
for (String type : types) {
perTypeValues[i++] = getOrdinalsValues(type);
}
}
@Override
public boolean advanceExact(int docId) throws IOException {
count = 0;
termsCursor = 0;
for (SortedDocValues values : perTypeValues) {
if (values.advanceExact(docId)) {
final int ord = values.ordValue();
terms[count++] = values.lookupOrd(ord);
}
}
assert count <= 2 : "A single doc can potentially be both parent and child, so the maximum allowed values is 2";
if (count > 1) {
int cmp = terms[0].compareTo(terms[1]);
if (cmp > 0) {
ArrayUtil.swap(terms, 0, 1);
} else if (cmp == 0) {
// If the id is the same between types the only omit one. For example: a doc has parent#1 in _uid field and has grand_parent#1 in _parent field.
count = 1;
}
}
return count != 0;
}
@Override
public int docValueCount() {
return count;
}
@Override
public BytesRef nextValue() throws IOException {
return terms[termsCursor++];
}
};
}
public static AtomicParentChildFieldData empty() {
return new AbstractAtomicParentChildFieldData() {
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public Collection<Accountable> getChildResources() {
return Collections.emptyList();
}
@Override
public void close() {
}
@Override
public SortedDocValues getOrdinalsValues(String type) {
return DocValues.emptySorted();
}
@Override
public Set<String> types() {
return emptySet();
}
};
}
}

6
core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
View File

@ -22,6 +22,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FilteredTermsEnum;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
@ -51,6 +52,11 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD
this.minSegmentSize = minSegmentSize;
}
@Override
public MultiDocValues.OrdinalMap getOrdinalMap() {
return null;
}
@Override
public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
if (indexReader.leaves().size() <= 1) {

401
core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
View File

@ -1,401 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LongValues;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.breaker.CircuitBreaker;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.fielddata.AbstractSortedDocValues;
import org.elasticsearch.index.fielddata.AtomicParentChildFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.IndexParentChildFieldData;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ParentFieldMapper;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
/**
* ParentChildIndexFieldData is responsible for loading the id cache mapping
* needed for has_child and has_parent queries into memory.
*/
public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicParentChildFieldData> implements IndexParentChildFieldData {
private final Set<String> parentTypes;
private final CircuitBreakerService breakerService;
public ParentChildIndexFieldData(IndexSettings indexSettings, String fieldName,
IndexFieldDataCache cache, MapperService mapperService,
CircuitBreakerService breakerService) {
super(indexSettings, fieldName, cache);
this.breakerService = breakerService;
Set<String> parentTypes = new HashSet<>();
for (DocumentMapper mapper : mapperService.docMappers(false)) {
ParentFieldMapper parentFieldMapper = mapper.parentFieldMapper();
if (parentFieldMapper.active()) {
parentTypes.add(parentFieldMapper.type());
}
}
this.parentTypes = parentTypes;
}
@Override
public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
final XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
return new SortField(getFieldName(), source, reverse);
}
@Override
public AtomicParentChildFieldData load(LeafReaderContext context) {
final LeafReader reader = context.reader();
return new AbstractAtomicParentChildFieldData() {
public Set<String> types() {
return parentTypes;
}
@Override
public SortedDocValues getOrdinalsValues(String type) {
try {
return DocValues.getSorted(reader, ParentFieldMapper.joinField(type));
} catch (IOException e) {
throw new IllegalStateException("cannot load join doc values field for type [" + type + "]", e);
}
}
@Override
public long ramBytesUsed() {
// unknown
return 0;
}
@Override
public Collection<Accountable> getChildResources() {
return Collections.emptyList();
}
@Override
public void close() throws ElasticsearchException {
}
};
}
@Override
public AbstractAtomicParentChildFieldData loadDirect(LeafReaderContext context) throws Exception {
throw new UnsupportedOperationException();
}
@Override
protected AtomicParentChildFieldData empty(int maxDoc) {
return AbstractAtomicParentChildFieldData.empty();
}
public static class Builder implements IndexFieldData.Builder {
@Override
public IndexFieldData<?> build(IndexSettings indexSettings,
MappedFieldType fieldType,
IndexFieldDataCache cache, CircuitBreakerService breakerService,
MapperService mapperService) {
return new ParentChildIndexFieldData(indexSettings, fieldType.name(), cache,
mapperService, breakerService);
}
}
@Override
public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) {
if (indexReader.leaves().size() <= 1) {
// ordinals are already global
return this;
}
try {
return cache.load(indexReader, this);
} catch (Exception e) {
if (e instanceof ElasticsearchException) {
throw (ElasticsearchException) e;
} else {
throw new ElasticsearchException(e);
}
}
}
private static OrdinalMap buildOrdinalMap(AtomicParentChildFieldData[] atomicFD, String parentType) throws IOException {
final SortedDocValues[] ordinals = new SortedDocValues[atomicFD.length];
for (int i = 0; i < ordinals.length; ++i) {
ordinals[i] = atomicFD[i].getOrdinalsValues(parentType);
}
return OrdinalMap.build(null, ordinals, PackedInts.DEFAULT);
}
private static class OrdinalMapAndAtomicFieldData {
final OrdinalMap ordMap;
final AtomicParentChildFieldData[] fieldData;
OrdinalMapAndAtomicFieldData(OrdinalMap ordMap, AtomicParentChildFieldData[] fieldData) {
this.ordMap = ordMap;
this.fieldData = fieldData;
}
}
@Override
public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
final long startTime = System.nanoTime();
long ramBytesUsed = 0;
final Map<String, OrdinalMapAndAtomicFieldData> perType = new HashMap<>();
for (String type : parentTypes) {
final AtomicParentChildFieldData[] fieldData = new AtomicParentChildFieldData[indexReader.leaves().size()];
for (LeafReaderContext context : indexReader.leaves()) {
fieldData[context.ord] = load(context);
}
final OrdinalMap ordMap = buildOrdinalMap(fieldData, type);
ramBytesUsed += ordMap.ramBytesUsed();
perType.put(type, new OrdinalMapAndAtomicFieldData(ordMap, fieldData));
}
final AtomicParentChildFieldData[] fielddata = new AtomicParentChildFieldData[indexReader.leaves().size()];
for (int i = 0; i < fielddata.length; ++i) {
fielddata[i] = new GlobalAtomicFieldData(parentTypes, perType, i);
}
breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(ramBytesUsed);
if (logger.isDebugEnabled()) {
logger.debug(
"global-ordinals [_parent] took [{}]",
new TimeValue(System.nanoTime() - startTime, TimeUnit.NANOSECONDS)
);
}
return new GlobalFieldData(indexReader, fielddata, ramBytesUsed, perType);
}
private static class GlobalAtomicFieldData extends AbstractAtomicParentChildFieldData {
private final Set<String> types;
private final Map<String, OrdinalMapAndAtomicFieldData> atomicFD;
private final int segmentIndex;
GlobalAtomicFieldData(Set<String> types, Map<String, OrdinalMapAndAtomicFieldData> atomicFD, int segmentIndex) {
this.types = types;
this.atomicFD = atomicFD;
this.segmentIndex = segmentIndex;
}
@Override
public Set<String> types() {
return types;
}
@Override
public SortedDocValues getOrdinalsValues(String type) {
final OrdinalMapAndAtomicFieldData atomicFD = this.atomicFD.get(type);
if (atomicFD == null) {
return DocValues.emptySorted();
}
final OrdinalMap ordMap = atomicFD.ordMap;
final SortedDocValues[] allSegmentValues = new SortedDocValues[atomicFD.fieldData.length];
for (int i = 0; i < allSegmentValues.length; ++i) {
allSegmentValues[i] = atomicFD.fieldData[i].getOrdinalsValues(type);
}
final SortedDocValues segmentValues = allSegmentValues[segmentIndex];
if (segmentValues.getValueCount() == ordMap.getValueCount()) {
// ords are already global
return segmentValues;
}
final LongValues globalOrds = ordMap.getGlobalOrds(segmentIndex);
return new AbstractSortedDocValues() {
@Override
public BytesRef lookupOrd(int ord) throws IOException {
final int segmentIndex = ordMap.getFirstSegmentNumber(ord);
final int segmentOrd = (int) ordMap.getFirstSegmentOrd(ord);
return allSegmentValues[segmentIndex].lookupOrd(segmentOrd);
}
@Override
public int getValueCount() {
return (int) ordMap.getValueCount();
}
@Override
public int ordValue() throws IOException {
return (int) globalOrds.get(segmentValues.ordValue());
}
@Override
public boolean advanceExact(int target) throws IOException {
return segmentValues.advanceExact(target);
}
@Override
public int docID() {
return segmentValues.docID();
}
};
}
@Override
public long ramBytesUsed() {
// this class does not take memory on its own, the index-level field data does
// it through the use of ordinal maps
return 0;
}
@Override
public Collection<Accountable> getChildResources() {
return Collections.emptyList();
}
@Override
public void close() {
List<Releasable> closeables = new ArrayList<>();
for (OrdinalMapAndAtomicFieldData fds : atomicFD.values()) {
closeables.addAll(Arrays.asList(fds.fieldData));
}
Releasables.close(closeables);
}
}
public class GlobalFieldData implements IndexParentChildFieldData, Accountable {
private final Object coreCacheKey;
private final List<LeafReaderContext> leaves;
private final AtomicParentChildFieldData[] fielddata;
private final long ramBytesUsed;
private final Map<String, OrdinalMapAndAtomicFieldData> ordinalMapPerType;
GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed, Map<String, OrdinalMapAndAtomicFieldData> ordinalMapPerType) {
this.coreCacheKey = reader.getReaderCacheHelper().getKey();
this.leaves = reader.leaves();
this.ramBytesUsed = ramBytesUsed;
this.fielddata = fielddata;
this.ordinalMapPerType = ordinalMapPerType;
}
@Override
public String getFieldName() {
return ParentChildIndexFieldData.this.getFieldName();
}
@Override
public AtomicParentChildFieldData load(LeafReaderContext context) {
assert context.reader().getCoreCacheHelper().getKey() == leaves.get(context.ord)
.reader().getCoreCacheHelper().getKey();
return fielddata[context.ord];
}
@Override
public AtomicParentChildFieldData loadDirect(LeafReaderContext context) throws Exception {
return load(context);
}
@Override
public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
throw new UnsupportedOperationException("No sorting on global ords");
}
@Override
public void clear() {
ParentChildIndexFieldData.this.clear();
}
@Override
public Index index() {
return ParentChildIndexFieldData.this.index();
}
@Override
public long ramBytesUsed() {
return ramBytesUsed;
}
@Override
public Collection<Accountable> getChildResources() {
return Collections.emptyList();
}
@Override
public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) {
if (indexReader.getReaderCacheHelper().getKey() == coreCacheKey) {
return this;
}
throw new IllegalStateException();
}
@Override
public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
return loadGlobal(indexReader);
}
}
/**
* Returns the global ordinal map for the specified type
*/
// TODO: OrdinalMap isn't expose in the field data framework, because it is an implementation detail.
// However the JoinUtil works directly with OrdinalMap, so this is a hack to get access to OrdinalMap
// I don't think we should expose OrdinalMap in IndexFieldData, because only parent/child relies on it and for the
// rest of the code OrdinalMap is an implementation detail, but maybe we can expose it in IndexParentChildFieldData interface?
public static MultiDocValues.OrdinalMap getOrdinalMap(IndexParentChildFieldData indexParentChildFieldData, String type) {
if (indexParentChildFieldData instanceof ParentChildIndexFieldData.GlobalFieldData) {
return ((GlobalFieldData) indexParentChildFieldData).ordinalMapPerType.get(type).ordMap;
} else {
// one segment, local ordinals are global
return null;
}
}
}

6
core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedSetSelector;
@ -125,4 +126,9 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i
public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger, scriptFunction);
}
@Override
public MultiDocValues.OrdinalMap getOrdinalMap() {
return null;
}
}

4
core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java
View File

@ -37,7 +37,7 @@ import org.elasticsearch.common.settings.loader.SettingsLoader;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
import org.elasticsearch.index.query.QueryShardContext;
import java.io.IOException;
@ -196,7 +196,7 @@ public class ParentFieldMapper extends MetadataFieldMapper {
@Override
public IndexFieldData.Builder fielddataBuilder() {
return new ParentChildIndexFieldData.Builder();
return new DocValuesIndexFieldData.Builder();
}
}

6
core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
View File

@ -23,7 +23,6 @@ import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.SnapshotDeletionPolicy;
@ -32,7 +31,6 @@ import org.apache.lucene.search.QueryCachingPolicy;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.UsageTrackingQueryCachingPolicy;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Lock;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.ThreadInterruptedException;
import org.elasticsearch.ElasticsearchException;
@ -921,9 +919,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
// That can be done out of mutex, since the engine can be closed half way.
Engine engine = getEngineOrNull();
if (engine == null) {
try (Lock ignored = store.directory().obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
return store.getMetadata(null);
}
return store.getMetadata(null, true);
}
}
indexCommit = deletionPolicy.snapshot();

41
core/src/main/java/org/elasticsearch/index/store/Store.java
View File

@ -99,6 +99,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
@ -240,7 +241,8 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
* {@link #readMetadataSnapshot(Path, ShardId, NodeEnvironment.ShardLocker, Logger)} to read a meta data while locking
* {@link IndexShard#snapshotStoreMetadata()} to safely read from an existing shard
* {@link IndexShard#acquireIndexCommit(boolean)} to get an {@link IndexCommit} which is safe to use but has to be freed
*
* @param commit the index commit to read the snapshot from or <code>null</code> if the latest snapshot should be read from the
* directory
* @throws CorruptIndexException if the lucene index is corrupted. This can be caused by a checksum mismatch or an
* unexpected exception when opening the index reading the segments file.
* @throws IndexFormatTooOldException if the lucene index is too old to be opened.
@ -250,16 +252,47 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
* @throws IndexNotFoundException if the commit point can't be found in this store
*/
public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException {
return getMetadata(commit, false);
}
/**
* Returns a new MetadataSnapshot for the given commit. If the given commit is <code>null</code>
* the latest commit point is used.
*
* Note that this method requires the caller verify it has the right to access the store and
* no concurrent file changes are happening. If in doubt, you probably want to use one of the following:
*
* {@link #readMetadataSnapshot(Path, ShardId, NodeEnvironment.ShardLocker, Logger)} to read a meta data while locking
* {@link IndexShard#snapshotStoreMetadata()} to safely read from an existing shard
* {@link IndexShard#acquireIndexCommit(boolean)} to get an {@link IndexCommit} which is safe to use but has to be freed
*
* @param commit the index commit to read the snapshot from or <code>null</code> if the latest snapshot should be read from the
* directory
* @param lockDirectory if <code>true</code> the index writer lock will be obtained before reading the snapshot. This should
* only be used if there is no started shard using this store.
* @throws CorruptIndexException if the lucene index is corrupted. This can be caused by a checksum mismatch or an
* unexpected exception when opening the index reading the segments file.
* @throws IndexFormatTooOldException if the lucene index is too old to be opened.
* @throws IndexFormatTooNewException if the lucene index is too new to be opened.
* @throws FileNotFoundException if one or more files referenced by a commit are not present.
* @throws NoSuchFileException if one or more files referenced by a commit are not present.
* @throws IndexNotFoundException if the commit point can't be found in this store
*/
public MetadataSnapshot getMetadata(IndexCommit commit, boolean lockDirectory) throws IOException {
ensureOpen();
failIfCorrupted();
metadataLock.readLock().lock();
try {
assert lockDirectory ? commit == null : true : "IW lock should not be obtained if there is a commit point available";
// if we lock the directory we also acquire the write lock since that makes sure that nobody else tries to lock the IW
// on this store at the same time.
java.util.concurrent.locks.Lock lock = lockDirectory ? metadataLock.writeLock() : metadataLock.readLock();
lock.lock();
try (Closeable ignored = lockDirectory ? directory.obtainLock(IndexWriter.WRITE_LOCK_NAME) : () -> {} ) {
return new MetadataSnapshot(commit, directory, logger);
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
markStoreCorrupted(ex);
throw ex;
} finally {
metadataLock.readLock().unlock();
lock.unlock();
}
}

36
core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
View File

@ -104,6 +104,7 @@ import org.elasticsearch.index.analysis.PersianNormalizationFilterFactory;
import org.elasticsearch.index.analysis.PorterStemTokenFilterFactory;
import org.elasticsearch.index.analysis.PortugueseAnalyzerProvider;
import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
import org.elasticsearch.index.analysis.PreConfiguredTokenizer;
import org.elasticsearch.index.analysis.ReverseTokenFilterFactory;
import org.elasticsearch.index.analysis.RomanianAnalyzerProvider;
import org.elasticsearch.index.analysis.RussianAnalyzerProvider;
@ -141,7 +142,6 @@ import org.elasticsearch.index.analysis.WhitespaceAnalyzerProvider;
import org.elasticsearch.index.analysis.WhitespaceTokenizerFactory;
import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory;
import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy;
import org.elasticsearch.plugins.AnalysisPlugin;
import java.io.IOException;
@ -178,9 +178,10 @@ public final class AnalysisModule {
NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> normalizers = setupNormalizers(plugins);
Map<String, PreConfiguredTokenFilter> preConfiguredTokenFilters = setupPreConfiguredTokenFilters(plugins);
Map<String, PreConfiguredTokenizer> preConfiguredTokenizers = setupPreConfiguredTokenizers(plugins);
analysisRegistry = new AnalysisRegistry(environment, charFilters.getRegistry(), tokenFilters.getRegistry(), tokenizers
.getRegistry(), analyzers.getRegistry(), normalizers.getRegistry(), preConfiguredTokenFilters);
.getRegistry(), analyzers.getRegistry(), normalizers.getRegistry(), preConfiguredTokenFilters, preConfiguredTokenizers);
}
HunspellService getHunspellService() {
@ -287,6 +288,37 @@ public final class AnalysisModule {
return unmodifiableMap(preConfiguredTokenFilters.getRegistry());
}
static Map<String, PreConfiguredTokenizer> setupPreConfiguredTokenizers(List<AnalysisPlugin> plugins) {
NamedRegistry<PreConfiguredTokenizer> preConfiguredTokenizers = new NamedRegistry<>("pre-configured tokenizer");
// Temporary shim to register old style pre-configured tokenizers
for (PreBuiltTokenizers tokenizer : PreBuiltTokenizers.values()) {
String name = tokenizer.name().toLowerCase(Locale.ROOT);
PreConfiguredTokenizer preConfigured;
switch (tokenizer.getCachingStrategy()) {
case ONE:
preConfigured = PreConfiguredTokenizer.singleton(name,
() -> tokenizer.create(Version.CURRENT), null);
break;
default:
throw new UnsupportedOperationException(
"Caching strategy unsupported by temporary shim [" + tokenizer + "]");
}
preConfiguredTokenizers.register(name, preConfigured);
}
// Temporary shim for aliases. TODO deprecate after they are moved
preConfiguredTokenizers.register("nGram", preConfiguredTokenizers.getRegistry().get("ngram"));
preConfiguredTokenizers.register("edgeNGram", preConfiguredTokenizers.getRegistry().get("edge_ngram"));
preConfiguredTokenizers.register("PathHierarchy", preConfiguredTokenizers.getRegistry().get("path_hierarchy"));
for (AnalysisPlugin plugin: plugins) {
for (PreConfiguredTokenizer tokenizer : plugin.getPreConfiguredTokenizers()) {
preConfiguredTokenizers.register(tokenizer.getName(), tokenizer);
}
}
return unmodifiableMap(preConfiguredTokenizers.getRegistry());
}
private NamedRegistry<AnalysisProvider<TokenizerFactory>> setupTokenizers(List<AnalysisPlugin> plugins) {
NamedRegistry<AnalysisProvider<TokenizerFactory>> tokenizers = new NamedRegistry<>("tokenizer");
tokenizers.register("standard", StandardTokenizerFactory::new);

123
core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenizers.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.indices.analysis;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.LetterTokenizer;
import org.apache.lucene.analysis.core.LowerCaseTokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.ngram.EdgeNGramTokenizer;
import org.apache.lucene.analysis.ngram.NGramTokenizer;
@ -33,6 +32,7 @@ import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
import org.apache.lucene.analysis.th.ThaiTokenizer;
import org.elasticsearch.Version;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.index.analysis.CustomNormalizerProvider;
import org.elasticsearch.index.analysis.MultiTermAwareComponent;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
@ -42,21 +42,21 @@ import java.util.Locale;
public enum PreBuiltTokenizers {
STANDARD(CachingStrategy.LUCENE) {
STANDARD(CachingStrategy.ONE) {
@Override
protected Tokenizer create(Version version) {
return new StandardTokenizer();
}
},
CLASSIC(CachingStrategy.LUCENE) {
CLASSIC(CachingStrategy.ONE) {
@Override
protected Tokenizer create(Version version) {
return new ClassicTokenizer();
}
},
UAX_URL_EMAIL(CachingStrategy.LUCENE) {
UAX_URL_EMAIL(CachingStrategy.ONE) {
@Override
protected Tokenizer create(Version version) {
return new UAX29URLEmailTokenizer();
@ -77,39 +77,28 @@ public enum PreBuiltTokenizers {
}
},
LETTER(CachingStrategy.LUCENE) {
LETTER(CachingStrategy.ONE) {
@Override
protected Tokenizer create(Version version) {
return new LetterTokenizer();
}
},
LOWERCASE(CachingStrategy.LUCENE) {
@Override
protected Tokenizer create(Version version) {
return new LowerCaseTokenizer();
}
@Override
protected TokenFilterFactory getMultiTermComponent(Version version) {
return PreBuiltTokenFilters.LOWERCASE.getTokenFilterFactory(version);
}
},
WHITESPACE(CachingStrategy.LUCENE) {
WHITESPACE(CachingStrategy.ONE) {
@Override
protected Tokenizer create(Version version) {
return new WhitespaceTokenizer();
}
},
NGRAM(CachingStrategy.LUCENE) {
NGRAM(CachingStrategy.ONE) {
@Override
protected Tokenizer create(Version version) {
return new NGramTokenizer();
}
},
EDGE_NGRAM(CachingStrategy.LUCENE) {
EDGE_NGRAM(CachingStrategy.ONE) {
@Override
protected Tokenizer create(Version version) {
return new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE);
@ -139,64 +128,60 @@ public enum PreBuiltTokenizers {
}
protected final PreBuiltCacheFactory.PreBuiltCache<TokenizerFactory> cache;
private final CachingStrategy cachingStrategy;
PreBuiltTokenizers(CachingStrategy cachingStrategy) {
this.cachingStrategy = cachingStrategy;
cache = PreBuiltCacheFactory.getCache(cachingStrategy);
}
public CachingStrategy getCachingStrategy() {
return cachingStrategy;
}
private interface MultiTermAwareTokenizerFactory extends TokenizerFactory, MultiTermAwareComponent {}
public synchronized TokenizerFactory getTokenizerFactory(final Version version) {
TokenizerFactory tokenizerFactory = cache.get(version);
if (tokenizerFactory == null) {
final String finalName = name().toLowerCase(Locale.ROOT);
if (getMultiTermComponent(version) != null) {
tokenizerFactory = new MultiTermAwareTokenizerFactory() {
@Override
public String name() {
return finalName;
}
@Override
public Tokenizer create() {
return PreBuiltTokenizers.this.create(version);
}
@Override
public Object getMultiTermComponent() {
return PreBuiltTokenizers.this.getMultiTermComponent(version);
}
};
} else {
tokenizerFactory = new TokenizerFactory() {
@Override
public String name() {
return finalName;
}
@Override
public Tokenizer create() {
return PreBuiltTokenizers.this.create(version);
}
};
}
cache.put(version, tokenizerFactory);
}
return tokenizerFactory;
}
/**
* Get a pre built Tokenizer by its name or fallback to the default one
* @param name Tokenizer name
* @param defaultTokenizer default Tokenizer if name not found
* Old style resolution for {@link TokenizerFactory}. Exists entirely to keep
* {@link CustomNormalizerProvider#build(java.util.Map, java.util.Map)} working during the migration.
*/
public static PreBuiltTokenizers getOrDefault(String name, PreBuiltTokenizers defaultTokenizer) {
try {
return valueOf(name.toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException e) {
return defaultTokenizer;
public synchronized TokenizerFactory getTokenizerFactory(final Version version) {
TokenizerFactory tokenizerFactory = cache.get(version);
if (tokenizerFactory == null) {
final String finalName = name().toLowerCase(Locale.ROOT);
if (getMultiTermComponent(version) != null) {
tokenizerFactory = new MultiTermAwareTokenizerFactory() {
@Override
public String name() {
return finalName;
}
@Override
public Tokenizer create() {
return PreBuiltTokenizers.this.create(version);
}
@Override
public Object getMultiTermComponent() {
return PreBuiltTokenizers.this.getMultiTermComponent(version);
}
};
} else {
tokenizerFactory = new TokenizerFactory() {
@Override
public String name() {
return finalName;
}
@Override
public Tokenizer create() {
return PreBuiltTokenizers.this.create(version);
}
};
}
cache.put(version, tokenizerFactory);
}
return tokenizerFactory;
}
}
}

14
core/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java
View File

@ -22,24 +22,21 @@ package org.elasticsearch.plugins;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.elasticsearch.Version;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalyzerProvider;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.PreConfiguredTokenizer;
import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import org.elasticsearch.indices.analysis.PreBuiltCacheFactory;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
@ -95,12 +92,19 @@ public interface AnalysisPlugin {
}
/**
* Override to add additional pre-configured token filters.
* Override to add additional pre-configured {@link TokenFilter}s.
*/
default List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
return emptyList();
}
/**
* Override to add additional pre-configured {@link Tokenizer}.
*/
default List<PreConfiguredTokenizer> getPreConfiguredTokenizers() {
return emptyList();
}
/**
* Override to add additional hunspell {@link org.apache.lucene.analysis.hunspell.Dictionary}s.
*/

38
core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
View File

@ -22,7 +22,6 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.IndexSearcher;
@ -32,18 +31,15 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.ScorerAware;
import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues;
import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.index.fielddata.AtomicParentChildFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.fielddata.IndexParentChildFieldData;
import org.elasticsearch.index.fielddata.MultiGeoPointValues;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.index.fielddata.SortingBinaryDocValues;
import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
import org.elasticsearch.script.LeafSearchScript;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.aggregations.support.ValuesSource.WithScript.BytesValues;
@ -150,40 +146,6 @@ public abstract class ValuesSource {
}
}
public static class ParentChild extends Bytes {
protected final ParentChildIndexFieldData indexFieldData;
public ParentChild(ParentChildIndexFieldData indexFieldData) {
this.indexFieldData = indexFieldData;
}
public long globalMaxOrd(IndexSearcher indexSearcher, String type) {
DirectoryReader indexReader = (DirectoryReader) indexSearcher.getIndexReader();
if (indexReader.leaves().isEmpty()) {
return 0;
} else {
LeafReaderContext atomicReaderContext = indexReader.leaves().get(0);
IndexParentChildFieldData globalFieldData = indexFieldData.loadGlobal(indexReader);
AtomicParentChildFieldData afd = globalFieldData.load(atomicReaderContext);
SortedDocValues values = afd.getOrdinalsValues(type);
return values.getValueCount();
}
}
public SortedDocValues globalOrdinalsValues(String type, LeafReaderContext context) {
final IndexParentChildFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader());
final AtomicParentChildFieldData atomicFieldData = global.load(context);
return atomicFieldData.getOrdinalsValues(type);
}
@Override
public SortedBinaryDocValues bytesValues(LeafReaderContext context) {
final AtomicParentChildFieldData atomicFieldData = indexFieldData.load(context);
return atomicFieldData.getBytesValues();
}
}
public static class FieldData extends Bytes {
protected final IndexFieldData<?> indexFieldData;

5
core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
View File

@ -26,7 +26,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.script.Script;
@ -317,9 +316,7 @@ public class ValuesSourceConfig<VS extends ValuesSource> {
private ValuesSource bytesField() throws IOException {
final IndexFieldData<?> indexFieldData = fieldContext().indexFieldData();
ValuesSource dataSource;
if (indexFieldData instanceof ParentChildIndexFieldData) {
dataSource = new ValuesSource.Bytes.WithOrdinals.ParentChild((ParentChildIndexFieldData) indexFieldData);
} else if (indexFieldData instanceof IndexOrdinalsFieldData) {
if (indexFieldData instanceof IndexOrdinalsFieldData) {
dataSource = new ValuesSource.Bytes.WithOrdinals.FieldData((IndexOrdinalsFieldData) indexFieldData);
} else {
dataSource = new ValuesSource.Bytes.FieldData(indexFieldData);

13
core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryBuilder;
@ -69,7 +70,7 @@ import java.util.Objects;
*
* @see org.elasticsearch.action.search.SearchRequest#source(SearchSourceBuilder)
*/
public final class SearchSourceBuilder extends ToXContentToBytes implements Writeable {
public final class SearchSourceBuilder extends ToXContentToBytes implements Writeable, ToXContentObject {
private static final DeprecationLogger DEPRECATION_LOGGER =
new DeprecationLogger(Loggers.getLogger(SearchSourceBuilder.class));
@ -1098,12 +1099,6 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
innerToXContent(builder, params);
builder.endObject();
return builder;
}
public void innerToXContent(XContentBuilder builder, Params params) throws IOException {
if (from != -1) {
builder.field(FROM_FIELD.getPreferredName(), from);
}
@ -1199,7 +1194,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
if (aggregations != null) {
builder.field(AGGREGATIONS_FIELD.getPreferredName(), aggregations);
}
}
if (highlightBuilder != null) {
builder.field(HIGHLIGHT_FIELD.getPreferredName(), highlightBuilder);
@ -1232,6 +1227,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
if (collapse != null) {
builder.field(COLLAPSE.getPreferredName(), collapse);
}
builder.endObject();
return builder;
}
public static class IndexBoost implements Writeable, ToXContent {

4
core/src/main/resources/org/elasticsearch/bootstrap/security.policy
View File

@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.1.jar}" {
//// Very special jar permissions:
//// These are dangerous permissions that we don't want to grant to everything.
grant codeBase "${codebase.lucene-core-7.0.0-snapshot-89f6d17.jar}" {
grant codeBase "${codebase.lucene-core-7.0.0-snapshot-a0aef2f.jar}" {
// needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
// java 8 package
permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-7.0.0-snapshot-89f6d17.jar}" {
permission java.lang.RuntimePermission "accessDeclaredMembers";
};
grant codeBase "${codebase.lucene-misc-7.0.0-snapshot-89f6d17.jar}" {
grant codeBase "${codebase.lucene-misc-7.0.0-snapshot-a0aef2f.jar}" {
// needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
permission java.nio.file.LinkPermission "hard";
};

2
core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
View File

@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
};
grant codeBase "${codebase.lucene-test-framework-7.0.0-snapshot-89f6d17.jar}" {
grant codeBase "${codebase.lucene-test-framework-7.0.0-snapshot-a0aef2f.jar}" {
// needed by RamUsageTester
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
// needed for testing hardlinks in StoreRecoveryTests since we install MockFS

3
core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
View File

@ -123,7 +123,8 @@ public class IndexModuleTests extends ESTestCase {
indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
index = indexSettings.getIndex();
environment = new Environment(settings);
emptyAnalysisRegistry = new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap());
emptyAnalysisRegistry = new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(),
emptyMap(), emptyMap());
threadPool = new TestThreadPool("test");
circuitBreakerService = new NoneCircuitBreakerService();
bigArrays = new BigArrays(settings, circuitBreakerService);

51
core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java
View File

@ -41,7 +41,6 @@ import org.elasticsearch.test.VersionUtils;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
@ -50,25 +49,29 @@ import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
public class AnalysisRegistryTests extends ESTestCase {
private Environment emptyEnvironment;
private AnalysisRegistry emptyRegistry;
private IndexSettings emptyIndexSettingsOfCurrentVersion;
private static AnalyzerProvider<?> analyzerProvider(final String name) {
return new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDEX, new EnglishAnalyzer());
}
private static AnalysisRegistry emptyAnalysisRegistry(Settings settings) {
return new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(),
emptyMap());
}
private static IndexSettings indexSettingsOfCurrentVersion(Settings.Builder settings) {
return IndexSettingsModule.newIndexSettings("index", settings
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build());
}
@Override
public void setUp() throws Exception {
super.setUp();
emptyEnvironment = new Environment(Settings.builder()
emptyRegistry = emptyAnalysisRegistry(Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build());
emptyRegistry = new AnalysisRegistry(emptyEnvironment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap());
emptyIndexSettingsOfCurrentVersion = IndexSettingsModule.newIndexSettings("index", Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build());
}
public void testDefaultAnalyzers() throws IOException {
@ -191,12 +194,8 @@ public class AnalysisRegistryTests extends ESTestCase {
Settings indexSettings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
IndexAnalyzers indexAnalyzers =
new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap())
.build(idxSettings);
IndexAnalyzers otherIndexAnalyzers =
new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap())
.build(idxSettings);
IndexAnalyzers indexAnalyzers = emptyAnalysisRegistry(settings).build(idxSettings);
IndexAnalyzers otherIndexAnalyzers = emptyAnalysisRegistry(settings).build(idxSettings);
final int numIters = randomIntBetween(5, 20);
for (int i = 0; i < numIters; i++) {
PreBuiltAnalyzers preBuiltAnalyzers = RandomPicks.randomFrom(random(), PreBuiltAnalyzers.values());
@ -204,22 +203,6 @@ public class AnalysisRegistryTests extends ESTestCase {
}
}
public void testPreConfiguredTokenFiltersAreCached() throws IOException {
AtomicBoolean built = new AtomicBoolean(false);
PreConfiguredTokenFilter assertsBuiltOnce = PreConfiguredTokenFilter.singleton("asserts_built_once", false, tokenStream -> {
if (false == built.compareAndSet(false, true)) {
fail("Attempted to build the token filter twice when it should have been cached");
}
return new MockTokenFilter(tokenStream, MockTokenFilter.EMPTY_STOPSET);
});
try (AnalysisRegistry registryWithPreBuiltTokenFilter = new AnalysisRegistry(emptyEnvironment, emptyMap(), emptyMap(), emptyMap(),
emptyMap(), emptyMap(), singletonMap("asserts_built_once", assertsBuiltOnce))) {
IndexAnalyzers indexAnalyzers = registryWithPreBuiltTokenFilter.build(emptyIndexSettingsOfCurrentVersion);
IndexAnalyzers otherIndexAnalyzers = registryWithPreBuiltTokenFilter.build(emptyIndexSettingsOfCurrentVersion);
assertSame(indexAnalyzers.get("asserts_built_once"), otherIndexAnalyzers.get("asserts_built_once"));
}
}
public void testNoTypeOrTokenizerErrorMessage() throws IOException {
Version version = VersionUtils.randomVersion(random());
Settings settings = Settings
@ -231,14 +214,12 @@ public class AnalysisRegistryTests extends ESTestCase {
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap())
.build(idxSettings));
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> emptyAnalysisRegistry(settings).build(idxSettings));
assertThat(e.getMessage(), equalTo("analyzer [test_analyzer] must specify either an analyzer type, or a tokenizer"));
}
public void testCloseIndexAnalyzersMultipleTimes() throws IOException {
IndexAnalyzers indexAnalyzers = emptyRegistry.build(emptyIndexSettingsOfCurrentVersion);
IndexAnalyzers indexAnalyzers = emptyRegistry.build(indexSettingsOfCurrentVersion(Settings.builder()));
indexAnalyzers.close();
indexAnalyzers.close();
}

2
core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
View File

@ -457,6 +457,7 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI
refreshReader();
IndexOrdinalsFieldData ifd = getForField("string", "value", hasDocValues());
IndexOrdinalsFieldData globalOrdinals = ifd.loadGlobal(topLevelReader);
assertNotNull(globalOrdinals.getOrdinalMap());
assertThat(topLevelReader.leaves().size(), equalTo(3));
// First segment
@ -584,6 +585,7 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI
refreshReader();
IndexOrdinalsFieldData ifd = getForField("string", "value", hasDocValues());
IndexOrdinalsFieldData globalOrdinals = ifd.loadGlobal(topLevelReader);
assertNotNull(globalOrdinals.getOrdinalMap());
assertThat(ifd.loadGlobal(topLevelReader), sameInstance(globalOrdinals));
// 3 b/c 1 segment level caches and 1 top level cache
// in case of doc values, we don't cache atomic FD, so only the top-level cache is there

278
core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java
View File

@ -1,278 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ParentFieldMapper;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.UidFieldMapper;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.search.MultiValueMode;
import org.junit.Before;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicReference;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
public class ParentChildFieldDataTests extends AbstractFieldDataTestCase {
private final String parentType = "parent";
private final String childType = "child";
private final String grandChildType = "grand-child";
@Before
public void setupData() throws Exception {
mapperService.merge(
childType, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(childType, "_parent", "type=" + parentType).string()), MapperService.MergeReason.MAPPING_UPDATE, false
);
mapperService.merge(
grandChildType, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(grandChildType, "_parent", "type=" + childType).string()), MapperService.MergeReason.MAPPING_UPDATE, false
);
Document d = new Document();
d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO));
d.add(createJoinField(parentType, "1"));
writer.addDocument(d);
d = new Document();
d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "2"), Field.Store.NO));
d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO));
d.add(createJoinField(parentType, "1"));
d.add(createJoinField(childType, "2"));
writer.addDocument(d);
writer.commit();
d = new Document();
d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "3"), Field.Store.NO));
d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO));
d.add(createJoinField(parentType, "1"));
d.add(createJoinField(childType, "3"));
writer.addDocument(d);
d = new Document();
d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(parentType, "2"), Field.Store.NO));
d.add(createJoinField(parentType, "2"));
writer.addDocument(d);
d = new Document();
d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "4"), Field.Store.NO));
d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "2"), Field.Store.NO));
d.add(createJoinField(parentType, "2"));
d.add(createJoinField(childType, "4"));
writer.addDocument(d);
d = new Document();
d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "5"), Field.Store.NO));
d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO));
d.add(createJoinField(parentType, "1"));
d.add(createJoinField(childType, "5"));
writer.addDocument(d);
writer.commit();
d = new Document();
d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(grandChildType, "6"), Field.Store.NO));
d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(childType, "2"), Field.Store.NO));
d.add(createJoinField(childType, "2"));
writer.addDocument(d);
d = new Document();
d.add(new StringField(UidFieldMapper.NAME, Uid.createUid("other-type", "1"), Field.Store.NO));
writer.addDocument(d);
}
private SortedDocValuesField createJoinField(String parentType, String id) {
return new SortedDocValuesField(ParentFieldMapper.joinField(parentType), new BytesRef(id));
}
public void testGetBytesValues() throws Exception {
writer.forceMerge(1); // force merge to 1 segment so we can iterate through documents
IndexFieldData indexFieldData = getForField(childType);
List<LeafReaderContext> readerContexts = refreshReader();
for (LeafReaderContext readerContext : readerContexts) {
AtomicFieldData fieldData = indexFieldData.load(readerContext);
SortedBinaryDocValues bytesValues = fieldData.getBytesValues();
assertTrue(bytesValues.advanceExact(0));
assertThat(bytesValues.docValueCount(), equalTo(1));
assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1"));
assertTrue(bytesValues.advanceExact(1));
assertThat(bytesValues.docValueCount(), equalTo(2));
assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1"));
assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2"));
assertTrue(bytesValues.advanceExact(2));
assertThat(bytesValues.docValueCount(), equalTo(2));
assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1"));
assertThat(bytesValues.nextValue().utf8ToString(), equalTo("3"));
assertTrue(bytesValues.advanceExact(3));
assertThat(bytesValues.docValueCount(), equalTo(1));
assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2"));
assertTrue(bytesValues.advanceExact(4));
assertThat(bytesValues.docValueCount(), equalTo(2));
assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2"));
assertThat(bytesValues.nextValue().utf8ToString(), equalTo("4"));
assertTrue(bytesValues.advanceExact(5));
assertThat(bytesValues.docValueCount(), equalTo(2));
assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1"));
assertThat(bytesValues.nextValue().utf8ToString(), equalTo("5"));
assertTrue(bytesValues.advanceExact(6));
assertThat(bytesValues.docValueCount(), equalTo(1));
assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2"));
assertFalse(bytesValues.advanceExact(7));
}
}
public void testSorting() throws Exception {
IndexFieldData indexFieldData = getForField(parentType);
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer));
SortField sortField = indexFieldData.sortField("_last", MultiValueMode.MIN, null, false);
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField));
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("1"));
assertThat(topDocs.scoreDocs[1].doc, equalTo(1));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("1"));
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("1"));
assertThat(topDocs.scoreDocs[3].doc, equalTo(5));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("1"));
assertThat(topDocs.scoreDocs[4].doc, equalTo(3));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("2"));
assertThat(topDocs.scoreDocs[5].doc, equalTo(4));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("2"));
assertThat(topDocs.scoreDocs[6].doc, equalTo(6));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("2"));
assertThat(topDocs.scoreDocs[7].doc, equalTo(7));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]), equalTo(null));
sortField = indexFieldData.sortField("_last", MultiValueMode.MIN, null, true);
topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField));
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(3));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("2"));
assertThat(topDocs.scoreDocs[1].doc, equalTo(4));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("2"));
assertThat(topDocs.scoreDocs[2].doc, equalTo(6));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("2"));
assertThat(topDocs.scoreDocs[3].doc, equalTo(0));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("1"));
assertThat(topDocs.scoreDocs[4].doc, equalTo(1));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("1"));
assertThat(topDocs.scoreDocs[5].doc, equalTo(2));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("1"));
assertThat(topDocs.scoreDocs[6].doc, equalTo(5));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("1"));
assertThat(topDocs.scoreDocs[7].doc, equalTo(7));
assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], nullValue());
}
public void testThreads() throws Exception {
final ParentChildIndexFieldData indexFieldData = getForField(childType);
final DirectoryReader reader = ElasticsearchDirectoryReader.wrap(
DirectoryReader.open(writer), new ShardId(new Index("test", ""), 0));
final IndexParentChildFieldData global = indexFieldData.loadGlobal(reader);
final AtomicReference<Exception> error = new AtomicReference<>();
final int numThreads = scaledRandomIntBetween(3, 8);
final Thread[] threads = new Thread[numThreads];
final CountDownLatch latch = new CountDownLatch(1);
final Map<Object, BytesRef[]> expected = new HashMap<>();
for (LeafReaderContext context : reader.leaves()) {
AtomicParentChildFieldData leafData = global.load(context);
SortedDocValues parentIds = leafData.getOrdinalsValues(parentType);
final BytesRef[] ids = new BytesRef[parentIds.getValueCount()];
for (int j = 0; j < parentIds.getValueCount(); ++j) {
final BytesRef id = parentIds.lookupOrd(j);
if (id != null) {
ids[j] = BytesRef.deepCopyOf(id);
}
}
expected.put(context.reader().getCoreCacheHelper().getKey(), ids);
}
for (int i = 0; i < numThreads; ++i) {
threads[i] = new Thread() {
@Override
public void run() {
try {
latch.await();
for (int i = 0; i < 100000; ++i) {
for (LeafReaderContext context : reader.leaves()) {
AtomicParentChildFieldData leafData = global.load(context);
SortedDocValues parentIds = leafData.getOrdinalsValues(parentType);
final BytesRef[] expectedIds = expected.get(context.reader().getCoreCacheHelper().getKey());
for (int j = 0; j < parentIds.getValueCount(); ++j) {
final BytesRef id = parentIds.lookupOrd(j);
assertEquals(expectedIds[j], id);
}
}
}
} catch (Exception e) {
error.compareAndSet(null, e);
}
}
};
threads[i].start();
}
latch.countDown();
for (Thread thread : threads) {
thread.join();
}
if (error.get() != null) {
throw error.get();
}
}
@Override
protected String getFieldDataType() {
return "_parent";
}
}

46
core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View File

@ -1498,6 +1498,52 @@ public class IndexShardTests extends IndexShardTestCase {
}
}
/**
* here we are simulating the scenario that happens when we do async shard fetching from GatewaySerivce while we are finishing
* a recovery and concurrently clean files. This should always be possible without any exception. Yet there was a bug where IndexShard
* acquired the index writer lock before it called into the store that has it's own locking for metadata reads
*/
public void testReadSnapshotConcurrently() throws IOException, InterruptedException {
IndexShard indexShard = newStartedShard();
indexDoc(indexShard, "doc", "0", "{\"foo\" : \"bar\"}");
if (randomBoolean()) {
indexShard.refresh("test");
}
indexDoc(indexShard, "doc", "1", "{\"foo\" : \"bar\"}");
indexShard.flush(new FlushRequest());
closeShards(indexShard);
final IndexShard newShard = reinitShard(indexShard);
Store.MetadataSnapshot storeFileMetaDatas = newShard.snapshotStoreMetadata();
assertTrue("at least 2 files, commit and data: " +storeFileMetaDatas.toString(), storeFileMetaDatas.size() > 1);
AtomicBoolean stop = new AtomicBoolean(false);
CountDownLatch latch = new CountDownLatch(1);
expectThrows(AlreadyClosedException.class, () -> newShard.getEngine()); // no engine
Thread thread = new Thread(() -> {
latch.countDown();
while(stop.get() == false){
try {
Store.MetadataSnapshot readMeta = newShard.snapshotStoreMetadata();
assertEquals(0, storeFileMetaDatas.recoveryDiff(readMeta).different.size());
assertEquals(0, storeFileMetaDatas.recoveryDiff(readMeta).missing.size());
assertEquals(storeFileMetaDatas.size(), storeFileMetaDatas.recoveryDiff(readMeta).identical.size());
} catch (IOException e) {
throw new AssertionError(e);
}
}
});
thread.start();
latch.await();
int iters = iterations(10, 100);
for (int i = 0; i < iters; i++) {
newShard.store().cleanupAndVerify("test", storeFileMetaDatas);
}
assertTrue(stop.compareAndSet(false, true));
thread.join();
closeShards(newShard);
}
/** A dummy repository for testing which just needs restore overridden */
private abstract static class RestoreOnlyRepository extends AbstractLifecycleComponent implements Repository {
private final String indexName;

150
core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
View File

@ -23,7 +23,6 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.hunspell.Dictionary;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@ -37,12 +36,12 @@ import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.Analysis;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.AnalysisTestsHelper;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.CustomAnalyzer;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
import org.elasticsearch.index.analysis.PreConfiguredTokenizer;
import org.elasticsearch.index.analysis.StandardTokenizerFactory;
import org.elasticsearch.index.analysis.StopTokenFilterFactory;
import org.elasticsearch.index.analysis.TokenFilterFactory;
@ -57,7 +56,6 @@ import org.hamcrest.MatcherAssert;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@ -164,18 +162,6 @@ public class AnalysisModuleTests extends ESTestCase {
assertEquals(org.apache.lucene.util.Version.fromBits(3,6,0), indexAnalyzers.get("custom7").analyzer().getVersion());
}
private void assertTokenFilter(String name, Class<?> clazz) throws IOException {
Settings settings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
TokenFilterFactory tokenFilter = analysis.tokenFilter.get(name);
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader("foo bar"));
TokenStream stream = tokenFilter.create(tokenizer);
assertThat(stream, instanceOf(clazz));
}
private void testSimpleConfiguration(Settings settings) throws IOException {
IndexAnalyzers indexAnalyzers = getIndexAnalyzers(settings);
Analyzer analyzer = indexAnalyzers.get("custom1").analyzer();
@ -269,27 +255,6 @@ public class AnalysisModuleTests extends ESTestCase {
* and that do not vary based on version at all.
*/
public void testPluginPreConfiguredTokenFilters() throws IOException {
// Simple token filter that appends text to the term
final class AppendTokenFilter extends TokenFilter {
private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
private final char[] appendMe;
protected AppendTokenFilter(TokenStream input, String appendMe) {
super(input);
this.appendMe = appendMe.toCharArray();
}
@Override
public boolean incrementToken() throws IOException {
if (false == input.incrementToken()) {
return false;
}
term.resizeBuffer(term.length() + appendMe.length);
System.arraycopy(appendMe, 0, term.buffer(), term.length(), appendMe.length);
term.setLength(term.length() + appendMe.length);
return true;
}
}
boolean noVersionSupportsMultiTerm = randomBoolean();
boolean luceneVersionSupportsMultiTerm = randomBoolean();
boolean elasticsearchVersionSupportsMultiTerm = randomBoolean();
@ -329,6 +294,82 @@ public class AnalysisModuleTests extends ESTestCase {
analyzers.get("elasticsearch_version").normalize("", "test").utf8ToString());
}
/**
* Tests that plugins can register pre-configured token filters that vary in behavior based on Elasticsearch version, Lucene version,
* and that do not vary based on version at all.
*/
public void testPluginPreConfiguredTokenizers() throws IOException {
boolean noVersionSupportsMultiTerm = randomBoolean();
boolean luceneVersionSupportsMultiTerm = randomBoolean();
boolean elasticsearchVersionSupportsMultiTerm = randomBoolean();
// Simple tokenizer that always spits out a single token with some preconfigured characters
final class FixedTokenizer extends Tokenizer {
private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
private final char[] chars;
private boolean read = false;
protected FixedTokenizer(String chars) {
this.chars = chars.toCharArray();
}
@Override
public boolean incrementToken() throws IOException {
if (read) {
return false;
}
clearAttributes();
read = true;
term.resizeBuffer(chars.length);
System.arraycopy(chars, 0, term.buffer(), 0, chars.length);
term.setLength(chars.length);
return true;
}
@Override
public void reset() throws IOException {
super.reset();
read = false;
}
}
AnalysisRegistry registry = new AnalysisModule(new Environment(emptyNodeSettings), singletonList(new AnalysisPlugin() {
@Override
public List<PreConfiguredTokenizer> getPreConfiguredTokenizers() {
return Arrays.asList(
PreConfiguredTokenizer.singleton("no_version", () -> new FixedTokenizer("no_version"),
noVersionSupportsMultiTerm ? () -> AppendTokenFilter.factoryForSuffix("no_version") : null),
PreConfiguredTokenizer.luceneVersion("lucene_version",
luceneVersion -> new FixedTokenizer(luceneVersion.toString()),
luceneVersionSupportsMultiTerm ?
luceneVersion -> AppendTokenFilter.factoryForSuffix(luceneVersion.toString()) : null),
PreConfiguredTokenizer.elasticsearchVersion("elasticsearch_version",
esVersion -> new FixedTokenizer(esVersion.toString()),
elasticsearchVersionSupportsMultiTerm ?
esVersion -> AppendTokenFilter.factoryForSuffix(esVersion.toString()) : null)
);
}
})).getAnalysisRegistry();
Version version = VersionUtils.randomVersion(random());
IndexAnalyzers analyzers = getIndexAnalyzers(registry, Settings.builder()
.put("index.analysis.analyzer.no_version.tokenizer", "no_version")
.put("index.analysis.analyzer.lucene_version.tokenizer", "lucene_version")
.put("index.analysis.analyzer.elasticsearch_version.tokenizer", "elasticsearch_version")
.put(IndexMetaData.SETTING_VERSION_CREATED, version)
.build());
assertTokenStreamContents(analyzers.get("no_version").tokenStream("", "test"), new String[] {"no_version"});
assertTokenStreamContents(analyzers.get("lucene_version").tokenStream("", "test"), new String[] {version.luceneVersion.toString()});
assertTokenStreamContents(analyzers.get("elasticsearch_version").tokenStream("", "test"), new String[] {version.toString()});
// These are current broken by https://github.com/elastic/elasticsearch/issues/24752
// assertEquals("test" + (noVersionSupportsMultiTerm ? "no_version" : ""),
// analyzers.get("no_version").normalize("", "test").utf8ToString());
// assertEquals("test" + (luceneVersionSupportsMultiTerm ? version.luceneVersion.toString() : ""),
// analyzers.get("lucene_version").normalize("", "test").utf8ToString());
// assertEquals("test" + (elasticsearchVersionSupportsMultiTerm ? version.toString() : ""),
// analyzers.get("elasticsearch_version").normalize("", "test").utf8ToString());
}
public void testRegisterHunspellDictionary() throws Exception {
Settings settings = Settings.builder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
@ -349,4 +390,41 @@ public class AnalysisModuleTests extends ESTestCase {
}));
assertSame(dictionary, module.getHunspellService().getDictionary("foo"));
}
// Simple token filter that appends text to the term
private static class AppendTokenFilter extends TokenFilter {
public static TokenFilterFactory factoryForSuffix(String suffix) {
return new TokenFilterFactory() {
@Override
public String name() {
return suffix;
}
@Override
public TokenStream create(TokenStream tokenStream) {
return new AppendTokenFilter(tokenStream, suffix);
}
};
}
private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
private final char[] appendMe;
protected AppendTokenFilter(TokenStream input, String appendMe) {
super(input);
this.appendMe = appendMe.toCharArray();
}
@Override
public boolean incrementToken() throws IOException {
if (false == input.incrementToken()) {
return false;
}
term.resizeBuffer(term.length() + appendMe.length);
System.arraycopy(appendMe, 0, term.buffer(), term.length(), appendMe.length);
term.setLength(term.length() + appendMe.length);
return true;
}
}
}

2
core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
View File

@ -63,7 +63,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
assertParseSearchSource(testSearchSourceBuilder, createParser(builder));
}
private void assertParseSearchSource(SearchSourceBuilder testBuilder, XContentParser parser) throws IOException {
private static void assertParseSearchSource(SearchSourceBuilder testBuilder, XContentParser parser) throws IOException {
QueryParseContext parseContext = new QueryParseContext(parser);
if (randomBoolean()) {
parser.nextToken(); // sometimes we move it on the START_OBJECT to

6
docs/painless/painless-getting-started.asciidoc
View File

@ -339,14 +339,14 @@ parameters and Java which uses the compile time types of the parameters.
The consequence of this that Painless doesn't support overloaded methods like
Java, leading to some trouble when it whitelists classes from the Java
standard library. For example, in Java and Groovy, `Matcher` has two methods:
`group(int)` and `group(String)`. Painless can't whitelist both of them methods
`group(int)` and `group(String)`. Painless can't whitelist both of these methods
because they have the same name and the same number of parameters. So instead it
has `group(int)` and `namedGroup(String)`.
We have a few justifications for this different way of dispatching methods:
1. It makes operating on `def` types simpler and, presumably, faster. Using
receiver, name, and arity means when Painless sees a call on a `def` objects it
receiver, name, and arity means that when Painless sees a call on a `def` object it
can dispatch the appropriate method without having to do expensive comparisons
of the types of the parameters. The same is true for invocations with `def`
typed parameters.
@ -357,4 +357,4 @@ otherwise. It'd be slow for it to behave like Groovy all the time.
dispatch *feels* like it'd add a ton of complexity which'd make maintenance and
other improvements much more difficult.
include::painless-debugging.asciidoc[]
include::painless-debugging.asciidoc[]

2
docs/reference/setup/configuration.asciidoc
View File

@ -51,7 +51,7 @@ path.logs: /var/log/elasticsearch
--------------------------------------------------
[float]
=== Environment variable subsitution
=== Environment variable substitution
Environment variables referenced with the `${...}` notation within the
configuration file will be replaced with the value of the environment

21
modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
View File

@ -20,7 +20,9 @@
package org.elasticsearch.analysis.common;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.ar.ArabicNormalizationFilter;
import org.apache.lucene.analysis.ar.ArabicStemFilter;
import org.apache.lucene.analysis.br.BrazilianStemFilter;
@ -29,6 +31,7 @@ import org.apache.lucene.analysis.cjk.CJKWidthFilter;
import org.apache.lucene.analysis.ckb.SoraniNormalizationFilter;
import org.apache.lucene.analysis.commongrams.CommonGramsFilter;
import org.apache.lucene.analysis.core.DecimalDigitFilter;
import org.apache.lucene.analysis.core.LowerCaseTokenizer;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.UpperCaseFilter;
import org.apache.lucene.analysis.cz.CzechStemFilter;
@ -66,6 +69,7 @@ import org.elasticsearch.index.analysis.DelimitedPayloadTokenFilterFactory;
import org.elasticsearch.index.analysis.HtmlStripCharFilterFactory;
import org.elasticsearch.index.analysis.LimitTokenCountFilterFactory;
import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
import org.elasticsearch.index.analysis.PreConfiguredTokenizer;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
import org.elasticsearch.plugins.AnalysisPlugin;
@ -174,4 +178,21 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin {
| WordDelimiterGraphFilter.STEM_ENGLISH_POSSESSIVE, null)));
return filters;
}
@Override
public List<PreConfiguredTokenizer> getPreConfiguredTokenizers() {
List<PreConfiguredTokenizer> tokenizers = new ArrayList<>();
tokenizers.add(PreConfiguredTokenizer.singleton("lowercase", LowerCaseTokenizer::new, () -> new TokenFilterFactory() {
@Override
public String name() {
return "lowercase";
}
@Override
public TokenStream create(TokenStream tokenStream) {
return new LowerCaseFilter(tokenStream);
}
}));
return tokenizers;
}
}

7
modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java
View File

@ -117,6 +117,13 @@ public class CommonAnalysisFactoryTests extends AnalysisFactoryTestCase {
return filters;
}
@Override
protected Map<String, Class<?>> getPreConfiguredTokenizers() {
Map<String, Class<?>> filters = new TreeMap<>(super.getPreConfiguredTokenFilters());
return filters;
}
/**
* Fails if a tokenizer is marked in the superclass with {@link MovedToAnalysisCommon} but
* hasn't been marked in this class with its proper factory.

1
modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
52bc12bbc30db614056896495f30699d69eabae4

1
modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e7bfe234a793f8a1f0556def4e526d040ed636c8

9
modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.script.expression;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Explanation;
import java.io.IOException;
@ -45,6 +46,14 @@ final class ReplaceableConstDoubleValueSource extends DoubleValuesSource {
return false;
}
@Override
public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
if (fv.advanceExact(docId))
return Explanation.match((float)fv.doubleValue(), "ReplaceableConstDoubleValues");
else
return Explanation.noMatch("ReplaceableConstDoubleValues");
}
@Override
public boolean equals(Object o) {
return o == this;

33
modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java
View File

@ -25,15 +25,16 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.ParentFieldMapper;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
import org.elasticsearch.search.aggregations.AggregatorFactory;
import org.elasticsearch.search.aggregations.support.FieldContext;
import org.elasticsearch.search.aggregations.support.ValueType;
import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.ParentChild;
import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.WithOrdinals;
import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
@ -43,10 +44,11 @@ import org.elasticsearch.search.internal.SearchContext;
import java.io.IOException;
import java.util.Objects;
public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder<ParentChild, ChildrenAggregationBuilder> {
public class ChildrenAggregationBuilder
extends ValuesSourceAggregationBuilder<WithOrdinals, ChildrenAggregationBuilder> {
public static final String NAME = "children";
private String parentType;
private final String childType;
private Query parentFilter;
private Query childFilter;
@ -79,15 +81,17 @@ public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder<P
}
@Override
protected ValuesSourceAggregatorFactory<ParentChild, ?> innerBuild(SearchContext context,
ValuesSourceConfig<ParentChild> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
return new ChildrenAggregatorFactory(name, config, parentType, childFilter, parentFilter, context, parent,
protected ValuesSourceAggregatorFactory<WithOrdinals, ?> innerBuild(SearchContext context,
ValuesSourceConfig<WithOrdinals> config,
AggregatorFactory<?> parent,
Builder subFactoriesBuilder) throws IOException {
return new ChildrenAggregatorFactory(name, config, childFilter, parentFilter, context, parent,
subFactoriesBuilder, metaData);
}
@Override
protected ValuesSourceConfig<ParentChild> resolveConfig(SearchContext context) {
ValuesSourceConfig<ParentChild> config = new ValuesSourceConfig<>(ValuesSourceType.BYTES);
protected ValuesSourceConfig<WithOrdinals> resolveConfig(SearchContext context) {
ValuesSourceConfig<WithOrdinals> config = new ValuesSourceConfig<>(ValuesSourceType.BYTES);
DocumentMapper childDocMapper = context.mapperService().documentMapper(childType);
if (childDocMapper != null) {
@ -95,15 +99,15 @@ public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder<P
if (!parentFieldMapper.active()) {
throw new IllegalArgumentException("[children] no [_parent] field not configured that points to a parent type");
}
parentType = parentFieldMapper.type();
String parentType = parentFieldMapper.type();
DocumentMapper parentDocMapper = context.mapperService().documentMapper(parentType);
if (parentDocMapper != null) {
parentFilter = parentDocMapper.typeFilter(context.getQueryShardContext());
childFilter = childDocMapper.typeFilter(context.getQueryShardContext());
ParentChildIndexFieldData parentChildIndexFieldData = context.fieldData()
.getForField(parentFieldMapper.fieldType());
config.fieldContext(new FieldContext(parentFieldMapper.fieldType().name(), parentChildIndexFieldData,
parentFieldMapper.fieldType()));
MappedFieldType parentFieldType = parentDocMapper.parentFieldMapper().getParentJoinFieldType();
final SortedSetDVOrdinalsIndexFieldData fieldData = context.fieldData().getForField(parentFieldType);
config.fieldContext(new FieldContext(parentFieldType.name(), fieldData,
parentFieldType));
} else {
config.unmapped(true);
}
@ -145,7 +149,6 @@ public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder<P
"Missing [child_type] field for children aggregation [" + aggregationName + "]");
}
return new ChildrenAggregationBuilder(aggregationName, childType);
}

22
modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java
View File

@ -26,8 +26,7 @@ import org.elasticsearch.search.aggregations.AggregatorFactory;
import org.elasticsearch.search.aggregations.InternalAggregation;
import org.elasticsearch.search.aggregations.NonCollectingAggregator;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import org.elasticsearch.search.aggregations.support.ValuesSource;
import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.ParentChild;
import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.WithOrdinals;
import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
import org.elasticsearch.search.internal.SearchContext;
@ -37,17 +36,15 @@ import java.util.List;
import java.util.Map;
public class ChildrenAggregatorFactory
extends ValuesSourceAggregatorFactory<ValuesSource.Bytes.WithOrdinals.ParentChild, ChildrenAggregatorFactory> {
extends ValuesSourceAggregatorFactory<WithOrdinals, ChildrenAggregatorFactory> {
private final String parentType;
private final Query parentFilter;
private final Query childFilter;
public ChildrenAggregatorFactory(String name, ValuesSourceConfig<ParentChild> config, String parentType, Query childFilter,
Query parentFilter, SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
Map<String, Object> metaData) throws IOException {
public ChildrenAggregatorFactory(String name, ValuesSourceConfig<WithOrdinals> config,
Query childFilter, Query parentFilter, SearchContext context, AggregatorFactory<?> parent,
AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
super(name, config, context, parent, subFactoriesBuilder, metaData);
this.parentType = parentType;
this.childFilter = childFilter;
this.parentFilter = parentFilter;
}
@ -66,12 +63,11 @@ public class ChildrenAggregatorFactory
}
@Override
protected Aggregator doCreateInternal(ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource, Aggregator parent,
protected Aggregator doCreateInternal(WithOrdinals valuesSource, Aggregator parent,
boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
throws IOException {
long maxOrd = valuesSource.globalMaxOrd(context.searcher(), parentType);
return new ParentToChildrenAggregator(name, factories, context, parent, parentType, childFilter, parentFilter, valuesSource, maxOrd,
pipelineAggregators, metaData);
long maxOrd = valuesSource.globalMaxOrd(context.searcher());
return new ParentToChildrenAggregator(name, factories, context, parent, childFilter,
parentFilter, valuesSource, maxOrd, pipelineAggregators, metaData);
}
}

23
modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java
View File

@ -20,7 +20,7 @@ package org.elasticsearch.join.aggregations;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Query;
@ -52,10 +52,9 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator {
static final ParseField TYPE_FIELD = new ParseField("type");
private final String parentType;
private final Weight childFilter;
private final Weight parentFilter;
private final ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource;
private final ValuesSource.Bytes.WithOrdinals valuesSource;
// Maybe use PagedGrowableWriter? This will be less wasteful than LongArray,
// but then we don't have the reuse feature of BigArrays.
@ -72,12 +71,11 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator {
private boolean multipleBucketsPerParentOrd = false;
public ParentToChildrenAggregator(String name, AggregatorFactories factories,
SearchContext context, Aggregator parent, String parentType, Query childFilter,
Query parentFilter, ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource,
SearchContext context, Aggregator parent, Query childFilter,
Query parentFilter, ValuesSource.Bytes.WithOrdinals valuesSource,
long maxOrd, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
throws IOException {
super(name, factories, context, parent, pipelineAggregators, metaData);
this.parentType = parentType;
// these two filters are cached in the parser
this.childFilter = context.searcher().createNormalizedWeight(childFilter, false);
this.parentFilter = context.searcher().createNormalizedWeight(parentFilter, false);
@ -105,9 +103,7 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator {
if (valuesSource == null) {
return LeafBucketCollector.NO_OP_COLLECTOR;
}
final SortedDocValues globalOrdinals = valuesSource.globalOrdinalsValues(parentType, ctx);
assert globalOrdinals != null;
final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx);
Scorer parentScorer = parentFilter.scorer(ctx);
final Bits parentDocs = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), parentScorer);
return new LeafBucketCollector() {
@ -115,7 +111,8 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator {
@Override
public void collect(int docId, long bucket) throws IOException {
if (parentDocs.get(docId) && globalOrdinals.advanceExact(docId)) {
long globalOrdinal = globalOrdinals.ordValue();
long globalOrdinal = globalOrdinals.nextOrd();
assert globalOrdinals.nextOrd() == SortedSetDocValues.NO_MORE_ORDS;
if (globalOrdinal != -1) {
if (parentOrdToBuckets.get(globalOrdinal) == -1) {
parentOrdToBuckets.set(globalOrdinal, bucket);
@ -147,9 +144,8 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator {
DocIdSetIterator childDocsIter = childDocsScorer.iterator();
final LeafBucketCollector sub = collectableSubAggregators.getLeafCollector(ctx);
final SortedDocValues globalOrdinals = valuesSource.globalOrdinalsValues(parentType,
ctx);
final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx);
// Set the scorer, since we now replay only the child docIds
sub.setScorer(new ConstantScoreScorer(null, 1f, childDocsIter));
@ -161,7 +157,8 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator {
continue;
}
if (globalOrdinals.advanceExact(docId)) {
long globalOrdinal = globalOrdinals.ordValue();
long globalOrdinal = globalOrdinals.nextOrd();
assert globalOrdinals.nextOrd() == SortedSetDocValues.NO_MORE_ORDS;
long bucketOrd = parentOrdToBuckets.get(globalOrdinal);
if (bucketOrd != -1) {
collectBucket(sub, docId, bucketOrd);

27
modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java
View File

@ -34,9 +34,10 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexParentChildFieldData;
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.ParentFieldMapper;
import org.elasticsearch.index.query.AbstractQueryBuilder;
import org.elasticsearch.index.query.InnerHitBuilder;
@ -48,7 +49,6 @@ import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import java.io.IOException;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
@ -324,9 +324,10 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
// wrap the query with type query
innerQuery = Queries.filtered(innerQuery, childDocMapper.typeFilter(context));
final ParentChildIndexFieldData parentChildIndexFieldData = context.getForField(parentFieldMapper.fieldType());
final MappedFieldType parentFieldType = parentDocMapper.parentFieldMapper().getParentJoinFieldType();
final SortedSetDVOrdinalsIndexFieldData fieldData = context.getForField(parentFieldType);
return new LateParsingQuery(parentDocMapper.typeFilter(context), innerQuery, minChildren(), maxChildren(),
parentType, scoreMode, parentChildIndexFieldData, context.getSearchSimilarity());
parentType, scoreMode, fieldData, context.getSearchSimilarity());
}
/**
@ -347,19 +348,19 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
private final int maxChildren;
private final String parentType;
private final ScoreMode scoreMode;
private final ParentChildIndexFieldData parentChildIndexFieldData;
private final SortedSetDVOrdinalsIndexFieldData fieldDataJoin;
private final Similarity similarity;
LateParsingQuery(Query toQuery, Query innerQuery, int minChildren, int maxChildren,
String parentType, ScoreMode scoreMode, ParentChildIndexFieldData parentChildIndexFieldData,
Similarity similarity) {
String parentType, ScoreMode scoreMode,
SortedSetDVOrdinalsIndexFieldData fieldData, Similarity similarity) {
this.toQuery = toQuery;
this.innerQuery = innerQuery;
this.minChildren = minChildren;
this.maxChildren = maxChildren;
this.parentType = parentType;
this.scoreMode = scoreMode;
this.parentChildIndexFieldData = parentChildIndexFieldData;
this.fieldDataJoin = fieldData;
this.similarity = similarity;
}
@ -374,10 +375,10 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
IndexSearcher indexSearcher = new IndexSearcher(reader);
indexSearcher.setQueryCache(null);
indexSearcher.setSimilarity(similarity);
IndexParentChildFieldData indexParentChildFieldData = parentChildIndexFieldData.loadGlobal((DirectoryReader) reader);
MultiDocValues.OrdinalMap ordinalMap = ParentChildIndexFieldData.getOrdinalMap(indexParentChildFieldData, parentType);
IndexOrdinalsFieldData indexParentChildFieldData = fieldDataJoin.loadGlobal((DirectoryReader) reader);
MultiDocValues.OrdinalMap ordinalMap = indexParentChildFieldData.getOrdinalMap();
return JoinUtil.createJoinQuery(joinField, innerQuery, toQuery, indexSearcher, scoreMode,
ordinalMap, minChildren, maxChildren);
ordinalMap, minChildren, maxChildren);
} else {
if (reader.leaves().isEmpty() && reader.numDocs() == 0) {
// asserting reader passes down a MultiReader during rewrite which makes this
@ -387,7 +388,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
return new MatchNoDocsQuery();
}
throw new IllegalStateException("can't load global ordinals for reader of type: " +
reader.getClass() + " must be a DirectoryReader");
reader.getClass() + " must be a DirectoryReader");
}
}

23
modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java
View File

@ -30,8 +30,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.ParentFieldMapper;
import org.elasticsearch.index.query.AbstractQueryBuilder;
import org.elasticsearch.index.query.InnerHitBuilder;
@ -176,15 +177,12 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
}
Set<String> childTypes = new HashSet<>();
ParentChildIndexFieldData parentChildIndexFieldData = null;
for (DocumentMapper documentMapper : context.getMapperService().docMappers(false)) {
ParentFieldMapper parentFieldMapper = documentMapper.parentFieldMapper();
if (parentFieldMapper.active() && type.equals(parentFieldMapper.type())) {
childTypes.add(documentMapper.type());
parentChildIndexFieldData = context.getForField(parentFieldMapper.fieldType());
}
}
if (childTypes.isEmpty()) {
throw new QueryShardException(context, "[" + NAME + "] no child types found for type [" + type + "]");
}
@ -204,14 +202,17 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
// wrap the query with type query
innerQuery = Queries.filtered(innerQuery, parentDocMapper.typeFilter(context));
final MappedFieldType parentType = parentDocMapper.parentFieldMapper().getParentJoinFieldType();
final SortedSetDVOrdinalsIndexFieldData fieldData = context.getForField(parentType);
return new HasChildQueryBuilder.LateParsingQuery(childrenQuery,
innerQuery,
HasChildQueryBuilder.DEFAULT_MIN_CHILDREN,
HasChildQueryBuilder.DEFAULT_MAX_CHILDREN,
type,
score ? ScoreMode.Max : ScoreMode.None,
parentChildIndexFieldData,
context.getSearchSimilarity());
innerQuery,
HasChildQueryBuilder.DEFAULT_MIN_CHILDREN,
HasChildQueryBuilder.DEFAULT_MAX_CHILDREN,
type,
score ? ScoreMode.Max : ScoreMode.None,
fieldData,
context.getSearchSimilarity());
}
@Override

3
modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java
View File

@ -173,7 +173,8 @@ public class ParentToChildrenAggregatorTests extends AggregatorTestCase {
private static ParentFieldMapper createParentFieldMapper() {
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
return new ParentFieldMapper.Builder("parent").type(PARENT_TYPE).build(new Mapper.BuilderContext(settings, new ContentPath(0)));
return new ParentFieldMapper.Builder("parent_type")
.type(PARENT_TYPE).build(new Mapper.BuilderContext(settings, new ContentPath(0)));
}
private void testCase(Query query, IndexSearcher indexSearcher, Consumer<InternalChildren> verify)

4
modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java
View File

@ -105,7 +105,7 @@ import static org.hamcrest.Matchers.notNullValue;
@ClusterScope(scope = Scope.SUITE)
public class ChildQuerySearchIT extends ESIntegTestCase {
@Override
protected boolean ignoreExternalCluster() {
return true;
@ -2008,7 +2008,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
.setParent("parent-id").setSource("searchText", "quick brown fox").get();
refresh();
String[] highlightTypes = new String[] {"plain", "fvh", "postings"};
String[] highlightTypes = new String[] {"plain", "fvh", "unified"};
for (String highlightType : highlightTypes) {
logger.info("Testing with highlight type [{}]", highlightType);
SearchResponse searchResponse = client().prepareSearch("test")

1
plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
e6e6d743c100e4d7bc55480d5333d634e41856ca

1
plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
18e2a8a8096b13e191882aa77134e27c68e60372

1
plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
e520a9c7f8a2fc9f7c575940d9b24834a592ca25

1
plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
236924d9d6da7e4f36535e957e9a506b4e737302

1
plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
32fe29deb1c10cb7ae70d5f4d95fcc414f9813d6

1
plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
f8b0087d03c65253122cbc3b3419f346204e80fe

1
plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
e50f4ab0d6ebf85b282a86707309343e3260c4a2

1
plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
3e5102270f6c10a3b33e402ed5f8722ec2a1a338

1
plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
a41c22ef7dd43991e1f3555ff527ac79eb47fdca

1
plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
6d9730ec654bdcf943a4018a5695e7954159ceda

5
plugins/analysis-ukrainian/build.gradle
View File

@ -24,8 +24,9 @@ esplugin {
dependencies {
compile "org.apache.lucene:lucene-analyzers-morfologik:${versions.lucene}"
compile "org.carrot2:morfologik-stemming:2.1.0"
compile "org.carrot2:morfologik-fsa:2.1.0"
compile "org.carrot2:morfologik-stemming:2.1.1"
compile "org.carrot2:morfologik-fsa:2.1.1"
compile "ua.net.nlp:morfologik-ukrainian-search:3.7.5"
}
dependencyLicenses {

1
plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-89f6d17.jar.sha1
View File

@ -1 +0,0 @@
01fe11b45d9f6a68ef1e9994bebd81d26632efc5

1
plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-a0aef2f.jar.sha1 Normal file
View File

@ -0,0 +1 @@
26d01ae0d15243b30874b2cb609be5d041890459

1
plugins/analysis-ukrainian/licenses/morfologik-fsa-2.1.0.jar.sha1
View File

@ -1 +0,0 @@
88e5993f73c102f378c711f6e47221b7a9e22d25

1
plugins/analysis-ukrainian/licenses/morfologik-fsa-2.1.1.jar.sha1 Normal file
View File

@ -0,0 +1 @@
87866deba6aa5d19956fbe3406d8ddb5f19f5352

1
plugins/analysis-ukrainian/licenses/morfologik-stemming-2.1.0.jar.sha1
View File

@ -1 +0,0 @@
94167b64752138a246cc33cbf1a3b0bfe5274b7c

1
plugins/analysis-ukrainian/licenses/morfologik-stemming-2.1.1.jar.sha1 Normal file
View File

@ -0,0 +1 @@
5c169bab2e7dd04f5cb03d179a73a4339cc1d0a2

1
plugins/analysis-ukrainian/licenses/morfologik-ukrainian-search-3.7.5.jar.sha1 Normal file
View File

@ -0,0 +1 @@
2b8c8fbd740164d220ca7d18605b8b2092e163e9

108
test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java
View File

@ -66,6 +66,7 @@ import org.elasticsearch.index.analysis.PatternTokenizerFactory;
import org.elasticsearch.index.analysis.PersianNormalizationFilterFactory;
import org.elasticsearch.index.analysis.PorterStemTokenFilterFactory;
import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
import org.elasticsearch.index.analysis.PreConfiguredTokenizer;
import org.elasticsearch.index.analysis.ReverseTokenFilterFactory;
import org.elasticsearch.index.analysis.ScandinavianFoldingFilterFactory;
import org.elasticsearch.index.analysis.ScandinavianNormalizationFilterFactory;
@ -95,6 +96,7 @@ import java.util.Collection;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@ -103,6 +105,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static java.util.Collections.singletonList;
import static org.hamcrest.Matchers.typeCompatibleWith;
/**
* Alerts us if new analysis components are added to Lucene, so we don't miss them.
@ -148,26 +151,6 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
.put("simplepatternsplit", Void.class)
.immutableMap();
static final Map<PreBuiltTokenizers, Class<?>> PREBUILT_TOKENIZERS;
static {
PREBUILT_TOKENIZERS = new EnumMap<>(PreBuiltTokenizers.class);
for (PreBuiltTokenizers tokenizer : PreBuiltTokenizers.values()) {
Class<?> luceneFactoryClazz;
switch (tokenizer) {
case UAX_URL_EMAIL:
luceneFactoryClazz = org.apache.lucene.analysis.standard.UAX29URLEmailTokenizerFactory.class;
break;
case PATH_HIERARCHY:
luceneFactoryClazz = Void.class;
break;
default:
luceneFactoryClazz = org.apache.lucene.analysis.util.TokenizerFactory.lookupClass(
toCamelCase(tokenizer.getTokenizerFactory(Version.CURRENT).name()));
}
PREBUILT_TOKENIZERS.put(tokenizer, luceneFactoryClazz);
}
}
static final Map<String,Class<?>> KNOWN_TOKENFILTERS = new MapBuilder<String,Class<?>>()
// exposed in ES
.put("apostrophe", ApostropheFilterFactory.class)
@ -319,22 +302,26 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
this.plugin = Objects.requireNonNull(plugin, "plugin is required. use an empty plugin for core");
}
protected Map<String, Class<?>> getTokenizers() {
return KNOWN_TOKENIZERS;
protected Map<String, Class<?>> getCharFilters() {
return KNOWN_CHARFILTERS;
}
protected Map<String, Class<?>> getTokenFilters() {
return KNOWN_TOKENFILTERS;
}
protected Map<String, Class<?>> getTokenizers() {
return KNOWN_TOKENIZERS;
}
/**
* Map containing pre-configured token filters that should be available
* after installing this plugin. The map is from the name of the token
* filter to the class of the Lucene {@link TokenFilterFactory} that it
* is emulating. If the Lucene filter factory is {@code null} then the
* test will look it up for you from the name. If there is no Lucene
* {@linkplain TokenFilterFactory} then the right hand side should
* be {@link Void}.
* is emulating. If the Lucene {@linkplain TokenFilterFactory} is
* {@code null} then the test will look it up for you from the name. If
* there is no Lucene {@linkplain TokenFilterFactory} then the right
* hand side should be {@link Void}.
*/
protected Map<String, Class<?>> getPreConfiguredTokenFilters() {
Map<String, Class<?>> filters = new HashMap<>();
@ -343,8 +330,33 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
return filters;
}
protected Map<String, Class<?>> getCharFilters() {
return KNOWN_CHARFILTERS;
/**
* Map containing pre-configured tokenizers that should be available
* after installing this plugin. The map is from the name of the token
* filter to the class of the Lucene {@link TokenizerFactory} that it
* is emulating. If the Lucene {@linkplain TokenizerFactory} is
* {@code null} then the test will look it up for you from the name.
* If there is no Lucene {@linkplain TokenizerFactory} then the right
* hand side should be {@link Void}.
*/
protected Map<String, Class<?>> getPreConfiguredTokenizers() {
Map<String, Class<?>> tokenizers = new HashMap<>();
// TODO drop this temporary shim when all the old style tokenizers have been migrated to new style
for (PreBuiltTokenizers tokenizer : PreBuiltTokenizers.values()) {
final Class<?> luceneFactoryClazz;
switch (tokenizer) {
case UAX_URL_EMAIL:
luceneFactoryClazz = org.apache.lucene.analysis.standard.UAX29URLEmailTokenizerFactory.class;
break;
case PATH_HIERARCHY:
luceneFactoryClazz = Void.class;
break;
default:
luceneFactoryClazz = null;
}
tokenizers.put(tokenizer.name().toLowerCase(Locale.ROOT), luceneFactoryClazz);
}
return tokenizers;
}
public void testTokenizers() {
@ -421,21 +433,8 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
Collection<Object> expected = new HashSet<>();
Collection<Object> actual = new HashSet<>();
for (Map.Entry<PreBuiltTokenizers, Class<?>> entry : PREBUILT_TOKENIZERS.entrySet()) {
PreBuiltTokenizers tokenizer = entry.getKey();
Class<?> luceneFactory = entry.getValue();
if (luceneFactory == Void.class) {
continue;
}
assertTrue(TokenizerFactory.class.isAssignableFrom(luceneFactory));
if (tokenizer.getTokenizerFactory(Version.CURRENT) instanceof MultiTermAwareComponent) {
actual.add(tokenizer);
}
if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom(luceneFactory)) {
expected.add(tokenizer);
}
}
Map<String, PreConfiguredTokenFilter> preBuiltTokenFilters = AnalysisModule.setupPreConfiguredTokenFilters(singletonList(plugin));
Map<String, PreConfiguredTokenFilter> preConfiguredTokenFilters =
AnalysisModule.setupPreConfiguredTokenFilters(singletonList(plugin));
for (Map.Entry<String, Class<?>> entry : getPreConfiguredTokenFilters().entrySet()) {
String name = entry.getKey();
Class<?> luceneFactory = entry.getValue();
@ -445,8 +444,8 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
if (luceneFactory == null) {
luceneFactory = TokenFilterFactory.lookupClass(toCamelCase(name));
}
assertTrue(TokenFilterFactory.class.isAssignableFrom(luceneFactory));
PreConfiguredTokenFilter filter = preBuiltTokenFilters.get(name);
assertThat(luceneFactory, typeCompatibleWith(TokenFilterFactory.class));
PreConfiguredTokenFilter filter = preConfiguredTokenFilters.get(name);
assertNotNull("test claims pre built token filter [" + name + "] should be available but it wasn't", filter);
if (filter.shouldUseFilterForMultitermQueries()) {
actual.add("token filter [" + name + "]");
@ -455,6 +454,25 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
expected.add("token filter [" + name + "]");
}
}
Map<String, PreConfiguredTokenizer> preConfiguredTokenizers = AnalysisModule.setupPreConfiguredTokenizers(singletonList(plugin));
for (Map.Entry<String, Class<?>> entry : getPreConfiguredTokenizers().entrySet()) {
String name = entry.getKey();
Class<?> luceneFactory = entry.getValue();
if (luceneFactory == Void.class) {
continue;
}
if (luceneFactory == null) {
luceneFactory = TokenizerFactory.lookupClass(toCamelCase(name));
}
assertThat(luceneFactory, typeCompatibleWith(TokenizerFactory.class));
PreConfiguredTokenizer tokenizer = preConfiguredTokenizers.get(name);
if (tokenizer.hasMultiTermComponent()) {
actual.add(tokenizer);
}
if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom(luceneFactory)) {
expected.add(tokenizer);
}
}
for (Map.Entry<PreBuiltCharFilters, Class<?>> entry : PREBUILT_CHARFILTERS.entrySet()) {
PreBuiltCharFilters charFilter = entry.getKey();
Class<?> luceneFactory = entry.getValue();