From 6dcd5bf61872748efb64c062425a887b6bff41fa Mon Sep 17 00:00:00 2001 From: gmarz Date: Mon, 11 May 2015 13:51:59 -0400 Subject: [PATCH 1/7] [TEST] Mlockall: Removing assertTrue until we have the proper test infra --- src/test/java/org/elasticsearch/common/jna/NativesTests.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test/java/org/elasticsearch/common/jna/NativesTests.java b/src/test/java/org/elasticsearch/common/jna/NativesTests.java index 37725b5b5e8..014a2ce5b27 100644 --- a/src/test/java/org/elasticsearch/common/jna/NativesTests.java +++ b/src/test/java/org/elasticsearch/common/jna/NativesTests.java @@ -32,10 +32,6 @@ public class NativesTests extends ElasticsearchTestCase { if (Constants.MAC_OS_X) { assertFalse("Memory locking is not available on OS X platforms", Natives.LOCAL_MLOCKALL); } - - if (Constants.WINDOWS) { - assertTrue(Natives.LOCAL_MLOCKALL); - } } @Test From e7618b8528736b96e84977795633166240f2127a Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 7 May 2015 23:05:12 -0700 Subject: [PATCH 2/7] Settings: Remove file based index templates As a follow up to #10870, this removes support for index templates on disk. It also removes a missed place still allowing disk based mappings. closes #11052 --- docs/reference/indices/templates.asciidoc | 38 +----- docs/reference/migration/migrate_2_0.asciidoc | 5 +- .../metadata/MetaDataCreateIndexService.java | 40 ------- .../index/mapper/FileBasedMappingsTests.java | 111 ------------------ .../IndexTemplateFileLoadingTests.java | 104 ---------------- 5 files changed, 5 insertions(+), 293 deletions(-) delete mode 100644 src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java delete mode 100644 src/test/java/org/elasticsearch/indices/template/IndexTemplateFileLoadingTests.java diff --git a/docs/reference/indices/templates.asciidoc b/docs/reference/indices/templates.asciidoc index 36d405c1ac5..3a5a3c5731b 100644 --- a/docs/reference/indices/templates.asciidoc +++ b/docs/reference/indices/templates.asciidoc @@ -68,7 +68,7 @@ curl -XDELETE localhost:9200/_template/template_1 [float] [[getting]] -=== GETting templates +=== Getting templates Index templates are identified by a name (in the above case `template_1`) and can be retrieved using the following: @@ -157,39 +157,3 @@ for indices of that start with `te*`, source will still be enabled. Note, for mappings, the merging is "deep", meaning that specific object/property based mappings can easily be added/overridden on higher order templates, with lower order templates providing the basis. - -[float] -[[config]] -=== Config - -Index templates can also be placed within the config location -(`path.conf`) under the `templates` directory (note, make sure to place -them on all master eligible nodes). For example, a file called -`template_1.json` can be placed under `config/templates` and it will be -added if it matches an index. Here is a sample of the mentioned file: - -[source,js] --------------------------------------------------- -{ - "template_1" : { - "template" : "*", - "settings" : { - "index.number_of_shards" : 2 - }, - "mappings" : { - "_default_" : { - "_source" : { - "enabled" : false - } - }, - "type1" : { - "_all" : { - "enabled" : false - } - } - } - } -} --------------------------------------------------- - -*Please note that templates added this way will not appear in the `/_template/*` API request.* diff --git a/docs/reference/migration/migrate_2_0.asciidoc b/docs/reference/migration/migrate_2_0.asciidoc index 06efc7a59af..0279d9d4761 100644 --- a/docs/reference/migration/migrate_2_0.asciidoc +++ b/docs/reference/migration/migrate_2_0.asciidoc @@ -519,4 +519,7 @@ Log messages are now truncated at 10,000 characters. This can be changed in the The `top_children` query has been removed in favour of the `has_child` query. The `top_children` query wasn't always faster than the `has_child` query and the `top_children` query was often inaccurate. The total hits and any aggregations in the -same search request will likely be off if `top_children` was used. \ No newline at end of file +same search request will likely be off if `top_children` was used. + +=== Removed file based index templates +Index templates can no longer be configured on disk. Use the `_template` API instead. diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 732561f66f1..1d0f584f618 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -300,22 +300,6 @@ public class MetaDataCreateIndexService extends AbstractComponent { } } - // now add config level mappings - Path mappingsDir = environment.configFile().resolve("mappings"); - if (Files.isDirectory(mappingsDir)) { - // first index level - Path indexMappingsDir = mappingsDir.resolve(request.index()); - if (Files.isDirectory(indexMappingsDir)) { - addMappings(mappings, indexMappingsDir); - } - - // second is the _default mapping - Path defaultMappingsDir = mappingsDir.resolve("_default"); - if (Files.isDirectory(defaultMappingsDir)) { - addMappings(mappings, defaultMappingsDir); - } - } - ImmutableSettings.Builder indexSettingsBuilder = settingsBuilder(); // apply templates, here, in reverse order, since first ones are better matching for (int i = templates.size() - 1; i >= 0; i--) { @@ -517,30 +501,6 @@ public class MetaDataCreateIndexService extends AbstractComponent { } } - // see if we have templates defined under config - final Path templatesDir = environment.configFile().resolve("templates"); - if (Files.isDirectory(templatesDir)) { - try (DirectoryStream stream = Files.newDirectoryStream(templatesDir)) { - for (Path templatesFile : stream) { - if (Files.isRegularFile(templatesFile)) { - XContentParser parser = null; - try { - final byte[] templatesData = Files.readAllBytes(templatesFile); - parser = XContentHelper.createParser(templatesData, 0, templatesData.length); - IndexTemplateMetaData template = IndexTemplateMetaData.Builder.fromXContent(parser, templatesFile.getFileName().toString()); - if (indexTemplateFilter.apply(request, template)) { - templates.add(template); - } - } catch (Exception e) { - logger.warn("[{}] failed to read template [{}] from config", e, request.index(), templatesFile.toAbsolutePath()); - } finally { - Releasables.closeWhileHandlingException(parser); - } - } - } - } - } - CollectionUtil.timSort(templates, new Comparator() { @Override public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) { diff --git a/src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java b/src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java deleted file mode 100644 index 6772a5dfcd4..00000000000 --- a/src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import com.google.common.collect.ImmutableSet; - -import org.apache.lucene.util.IOUtils; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; -import org.elasticsearch.test.ElasticsearchTestCase; - -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Map; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; - -public class FileBasedMappingsTests extends ElasticsearchTestCase { - - private static final String NAME = FileBasedMappingsTests.class.getSimpleName(); - - public void testFileBasedMappings() throws Exception { - Path configDir = createTempDir(); - Path mappingsDir = configDir.resolve("mappings"); - Path indexMappings = mappingsDir.resolve("index").resolve("type.json"); - Path defaultMappings = mappingsDir.resolve("_default").resolve("type.json"); - try { - Files.createDirectories(indexMappings.getParent()); - Files.createDirectories(defaultMappings.getParent()); - - try (OutputStream stream = Files.newOutputStream(indexMappings); - XContentBuilder builder = new XContentBuilder(JsonXContent.jsonXContent, stream)) { - builder.startObject() - .startObject("type") - .startObject("properties") - .startObject("f") - .field("type", "string") - .endObject() - .endObject() - .endObject() - .endObject(); - } - - try (OutputStream stream = Files.newOutputStream(defaultMappings); - XContentBuilder builder = new XContentBuilder(JsonXContent.jsonXContent, stream)) { - builder.startObject() - .startObject("type") - .startObject("properties") - .startObject("g") - .field("type", "string") - .endObject() - .endObject() - .endObject() - .endObject(); - } - - Settings settings = ImmutableSettings.builder() - .put(ClusterName.SETTING, NAME) - .put("node.name", NAME) - .put("path.home", createTempDir()) - .put("path.conf", configDir.toAbsolutePath()) - .put("http.enabled", false) - .build(); - - try (Node node = NodeBuilder.nodeBuilder().local(true).data(true).settings(settings).node()) { - - assertAcked(node.client().admin().indices().prepareCreate("index").addMapping("type", "h", "type=string").get()); - try { - final GetMappingsResponse response = node.client().admin().indices().prepareGetMappings("index").get(); - assertTrue(response.mappings().toString(), response.mappings().containsKey("index")); - MappingMetaData mappings = response.mappings().get("index").get("type"); - assertNotNull(mappings); - Map properties = (Map) (mappings.getSourceAsMap().get("properties")); - assertNotNull(properties); - assertEquals(ImmutableSet.of("f", "g", "h"), properties.keySet()); - } finally { - // remove the index... - assertAcked(node.client().admin().indices().prepareDelete("index")); - } - } - } finally { - IOUtils.rm(configDir); - } - } - -} diff --git a/src/test/java/org/elasticsearch/indices/template/IndexTemplateFileLoadingTests.java b/src/test/java/org/elasticsearch/indices/template/IndexTemplateFileLoadingTests.java deleted file mode 100644 index 86b8c3e8d1e..00000000000 --- a/src/test/java/org/elasticsearch/indices/template/IndexTemplateFileLoadingTests.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.indices.template; - -import com.carrotsearch.randomizedtesting.LifecycleScope; -import com.google.common.base.Charsets; - -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ElasticsearchIntegrationTest; -import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashSet; -import java.util.Locale; -import java.util.Set; - -import static org.elasticsearch.test.ElasticsearchIntegrationTest.*; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; - -@ClusterScope(scope= Scope.TEST, numDataNodes =1) -public class IndexTemplateFileLoadingTests extends ElasticsearchIntegrationTest { - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder(); - settingsBuilder.put(super.nodeSettings(nodeOrdinal)); - - try { - Path directory = createTempDir(); - settingsBuilder.put("path.conf", directory.toAbsolutePath()); - - Path templatesDir = directory.resolve("templates"); - Files.createDirectory(templatesDir); - - Path dst = templatesDir.resolve("template.json"); - String templatePath = "/org/elasticsearch/indices/template/template" + randomInt(5) + ".json"; - logger.info("Picking template path [{}]", templatePath); - // random template, one uses the 'setting.index.number_of_shards', the other 'settings.number_of_shards' - String template = Streams.copyToStringFromClasspath(templatePath); - Files.write(dst, template.getBytes(StandardCharsets.UTF_8)); - } catch (Exception e) { - throw new RuntimeException(e); - } - - return settingsBuilder.build(); - } - - @Override - protected int numberOfShards() { - //number of shards won't be set through index settings, the one from the index templates needs to be used - return -1; - } - - @Override - protected int numberOfReplicas() { - //number of replicas won't be set through index settings, the one from the index templates needs to be used - return -1; - } - - @Test - public void testThatLoadingTemplateFromFileWorks() throws Exception { - final int iters = scaledRandomIntBetween(1, 5); - Set indices = new HashSet<>(); - for (int i = 0; i < iters; i++) { - String indexName = "foo" + randomAsciiOfLengthBetween(0, 5).toLowerCase(Locale.ROOT); - if (indices.contains(indexName)) { - continue; - } - indices.add(indexName); - createIndex(indexName); - ensureYellow(); // ensuring yellow so the test fails faster if the template cannot be loaded - - ClusterStateResponse stateResponse = client().admin().cluster().prepareState().setIndices(indexName).get(); - assertThat(stateResponse.getState().getMetaData().indices().get(indexName).getNumberOfShards(), is(10)); - assertThat(stateResponse.getState().getMetaData().indices().get(indexName).getNumberOfReplicas(), is(0)); - assertThat(stateResponse.getState().getMetaData().indices().get(indexName).aliases().size(), equalTo(1)); - String aliasName = indexName + "-alias"; - assertThat(stateResponse.getState().getMetaData().indices().get(indexName).aliases().get(aliasName).alias(), equalTo(aliasName)); - } - } -} From 3ccdde7e2a98012c81e35155c9b63cb318bf57fb Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 11 May 2015 18:04:14 -0400 Subject: [PATCH 3/7] Generate access to tests paths like other paths. --- .../elasticsearch/bootstrap/security.policy | 12 ----------- .../elasticsearch/test/SecurityBootstrap.java | 20 +++++++++++++++++-- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/main/resources/org/elasticsearch/bootstrap/security.policy b/src/main/resources/org/elasticsearch/bootstrap/security.policy index 3f11557284a..3b252649beb 100644 --- a/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -32,18 +32,6 @@ grant codeBase "file:${{java.ext.dirs}}/*" { grant { - // system jar resources - permission java.io.FilePermission "${java.home}${/}-", "read"; - - // paths used for running tests - // compiled classes - permission java.io.FilePermission "${project.basedir}${/}target${/}classes${/}-", "read"; - permission java.io.FilePermission "${project.basedir}${/}target${/}test-classes${/}-", "read"; - // read permission for lib sigar - permission java.io.FilePermission "${project.basedir}${/}lib${/}sigar${/}-", "read"; - // mvn custom ./m2/repository for dependency jars - permission java.io.FilePermission "${m2.repository}${/}-", "read"; - permission java.nio.file.LinkPermission "symbolic"; permission groovy.security.GroovyCodeSourcePermission "/groovy/script"; diff --git a/src/test/java/org/elasticsearch/test/SecurityBootstrap.java b/src/test/java/org/elasticsearch/test/SecurityBootstrap.java index d5e050a0d3e..21ad1e7ac19 100644 --- a/src/test/java/org/elasticsearch/test/SecurityBootstrap.java +++ b/src/test/java/org/elasticsearch/test/SecurityBootstrap.java @@ -25,8 +25,10 @@ import org.elasticsearch.bootstrap.ESPolicy; import org.elasticsearch.bootstrap.Security; import org.elasticsearch.common.io.PathUtils; +import java.nio.file.Path; import java.security.Permissions; import java.security.Policy; +import java.util.Objects; import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean; @@ -48,9 +50,23 @@ class SecurityBootstrap { // install security manager if requested if (systemPropertyAsBoolean("tests.security.manager", false)) { try { - // initialize tmpdir the same exact way as bootstrap. + // initialize paths the same exact way as bootstrap. Permissions perms = new Permissions(); - Security.addPath(perms, PathUtils.get(System.getProperty("java.io.tmpdir")), "read,readlink,write,delete"); + Path basedir = PathUtils.get(Objects.requireNonNull(System.getProperty("project.basedir"), + "please set ${project.basedir} in pom.xml")); + // target/classes, target/test-classes + Security.addPath(perms, basedir.resolve("target").resolve("classes"), "read,readlink"); + Security.addPath(perms, basedir.resolve("target").resolve("test-classes"), "read,readlink"); + // lib/sigar + Security.addPath(perms, basedir.resolve("lib").resolve("sigar"), "read,readlink"); + // .m2/repository + Path m2repoDir = PathUtils.get(Objects.requireNonNull(System.getProperty("m2.repository"), + "please set ${m2.repository} in pom.xml")); + Security.addPath(perms, m2repoDir, "read,readlink"); + // java.io.tmpdir + Path javaTmpDir = PathUtils.get(Objects.requireNonNull(System.getProperty("java.io.tmpdir"), + "please set ${java.io.tmpdir} in pom.xml")); + Security.addPath(perms, javaTmpDir, "read,readlink,write,delete"); Policy.setPolicy(new ESPolicy(perms)); System.setSecurityManager(new TestSecurityManager()); Security.selfTest(); From 236f6ccad7ffb58b9f79bdcd810ca118ab331a9a Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 12 May 2015 09:32:12 +0200 Subject: [PATCH 4/7] [TEST] Reduce likelyness of sync on every operation --- src/test/java/org/elasticsearch/test/InternalTestCluster.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/elasticsearch/test/InternalTestCluster.java b/src/test/java/org/elasticsearch/test/InternalTestCluster.java index 1ce5b7e3066..505c9f49d48 100644 --- a/src/test/java/org/elasticsearch/test/InternalTestCluster.java +++ b/src/test/java/org/elasticsearch/test/InternalTestCluster.java @@ -441,7 +441,7 @@ public final class InternalTestCluster extends TestCluster { if (random.nextBoolean()) { builder.put(Translog.INDEX_TRANSLOG_FS_TYPE, RandomPicks.randomFrom(random, TranslogFile.Type.values())); - if (random.nextBoolean()) { + if (rarely(random)) { builder.put(Translog.INDEX_TRANSLOG_SYNC_INTERVAL, 0); // 0 has special meaning to sync each op } else { builder.put(Translog.INDEX_TRANSLOG_SYNC_INTERVAL, RandomInts.randomIntBetween(random, 100, 5000)); From 89b95dccc8eaf04927c307d76931236236b11d02 Mon Sep 17 00:00:00 2001 From: markharwood Date: Mon, 11 May 2015 16:18:59 +0100 Subject: [PATCH 5/7] Aggregations enhancement - remove pointless term frequency lookups. If the user has set a shard_min_doc_count setting then avoid looking up background frequencies if the term fails to meet the foreground threshold on a shard. Closes #11093 --- .../GlobalOrdinalsSignificantTermsAggregator.java | 8 +++++--- .../significant/SignificantLongTermsAggregator.java | 12 +++++++----- .../SignificantStringTermsAggregator.java | 13 ++++++++----- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java index 492167f1735..c46041dc29f 100644 --- a/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java @@ -99,6 +99,10 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri if (bucketCountThresholds.getMinDocCount() > 0 && bucketDocCount == 0) { continue; } + if (bucketDocCount < bucketCountThresholds.getShardMinDocCount()) { + continue; + } + if (spare == null) { spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null); } @@ -113,9 +117,7 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri // Back at the central reducer these properties will be updated with // global stats spare.updateScore(termsAggFactory.getSignificanceHeuristic()); - if (spare.subsetDf >= bucketCountThresholds.getShardMinDocCount()) { - spare = (SignificantStringTerms.Bucket) ordered.insertWithOverflow(spare); - } + spare = (SignificantStringTerms.Bucket) ordered.insertWithOverflow(spare); } final InternalSignificantTerms.Bucket[] list = new InternalSignificantTerms.Bucket[ordered.size()]; diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java index 329f5f566f5..0a33ab39a8e 100644 --- a/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java @@ -24,8 +24,8 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; import org.elasticsearch.search.aggregations.bucket.terms.LongTermsAggregator; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.aggregations.reducers.Reducer; @@ -82,11 +82,15 @@ public class SignificantLongTermsAggregator extends LongTermsAggregator { BucketSignificancePriorityQueue ordered = new BucketSignificancePriorityQueue(size); SignificantLongTerms.Bucket spare = null; for (long i = 0; i < bucketOrds.size(); i++) { + final int docCount = bucketDocCount(i); + if (docCount < bucketCountThresholds.getShardMinDocCount()) { + continue; + } if (spare == null) { spare = new SignificantLongTerms.Bucket(0, 0, 0, 0, 0, null, formatter); } spare.term = bucketOrds.get(i); - spare.subsetDf = bucketDocCount(i); + spare.subsetDf = docCount; spare.subsetSize = subsetSize; spare.supersetDf = termsAggFactory.getBackgroundFrequency(spare.term); spare.supersetSize = supersetSize; @@ -95,9 +99,7 @@ public class SignificantLongTermsAggregator extends LongTermsAggregator { spare.updateScore(termsAggFactory.getSignificanceHeuristic()); spare.bucketOrd = i; - if (spare.subsetDf >= bucketCountThresholds.getShardMinDocCount()) { - spare = (SignificantLongTerms.Bucket) ordered.insertWithOverflow(spare); - } + spare = (SignificantLongTerms.Bucket) ordered.insertWithOverflow(spare); } final InternalSignificantTerms.Bucket[] list = new InternalSignificantTerms.Bucket[ordered.size()]; diff --git a/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java b/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java index a49f18734ee..990a17ab7c9 100644 --- a/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java +++ b/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java @@ -24,8 +24,8 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; import org.elasticsearch.search.aggregations.bucket.terms.StringTermsAggregator; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; import org.elasticsearch.search.aggregations.reducers.Reducer; @@ -81,12 +81,17 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator { BucketSignificancePriorityQueue ordered = new BucketSignificancePriorityQueue(size); SignificantStringTerms.Bucket spare = null; for (int i = 0; i < bucketOrds.size(); i++) { + final int docCount = bucketDocCount(i); + if (docCount < bucketCountThresholds.getShardMinDocCount()) { + continue; + } + if (spare == null) { spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null); } bucketOrds.get(i, spare.termBytes); - spare.subsetDf = bucketDocCount(i); + spare.subsetDf = docCount; spare.subsetSize = subsetSize; spare.supersetDf = termsAggFactory.getBackgroundFrequency(spare.termBytes); spare.supersetSize = supersetSize; @@ -97,9 +102,7 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator { spare.updateScore(termsAggFactory.getSignificanceHeuristic()); spare.bucketOrd = i; - if (spare.subsetDf >= bucketCountThresholds.getShardMinDocCount()) { - spare = (SignificantStringTerms.Bucket) ordered.insertWithOverflow(spare); - } + spare = (SignificantStringTerms.Bucket) ordered.insertWithOverflow(spare); } final InternalSignificantTerms.Bucket[] list = new InternalSignificantTerms.Bucket[ordered.size()]; From 272babd9dc3042bf0112a71c6870b6c428d313ff Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Tue, 12 May 2015 10:24:12 +0200 Subject: [PATCH 6/7] [TEST] add comment bout security manager to bwc test description --- TESTING.asciidoc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/TESTING.asciidoc b/TESTING.asciidoc index aa14bc39e7b..e471bf858f2 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -224,18 +224,19 @@ mvn test -Dtests.jvm.argline="-XX:HeapDumpPath=/path/to/heapdumps" Running backwards compatibility tests is disabled by default since it requires a release version of elasticsearch to be present on the test system. -To run backwards compatibiilty tests untar or unzip a release and run the tests +To run backwards compatibilty tests untar or unzip a release and run the tests with the following command: --------------------------------------------------------------------------- -mvn test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.bwc.path=/path/to/elasticsearch +mvn test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.bwc.path=/path/to/elasticsearch -Dtests.security.manager=false --------------------------------------------------------------------------- +Note that backwards tests must be run with security manager disabled. If the elasticsearch release is placed under `./backwards/elasticsearch-x.y.z` the path can be omitted: --------------------------------------------------------------------------- -mvn test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z +mvn test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.security.manager=false --------------------------------------------------------------------------- To setup the bwc test environment execute the following steps (provided you are From 24ffcc28ac67dbd9ca6e27642121af17f7aaced7 Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Tue, 12 May 2015 14:39:42 +0200 Subject: [PATCH 7/7] fix merge conflict in javadoc --- .../java/org/elasticsearch/cluster/metadata/MetaData.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 97a1367d8e8..b11c5fbab7d 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -642,19 +642,11 @@ public class MetaData implements Iterable, Diffable { * @param indicesOptions how the aliases or indices need to be resolved to concrete indices * @param aliasesOrIndices the aliases or indices to be resolved to concrete indices * @return the obtained concrete indices -<<<<<<< HEAD * @throws IndexMissingException if one of the aliases or indices is missing and the provided indices options * don't allow such a case, or if the final result of the indices resolution is no indices and the indices options * don't allow such a case. * @throws IllegalArgumentException if one of the aliases resolve to multiple indices and the provided * indices options don't allow such a case. -======= - * @throws IndexMissingException if one of the aliases or indices is missing and the provided indices options - * don't allow such a case, or if the final result of the indices resolution is no indices and the indices options - * don't allow such a case. - * @throws ElasticsearchIllegalArgumentException if one of the aliases resolve to multiple indices and the provided - * indices options don't allow such a case. ->>>>>>> Add support for cluster state diffs */ public String[] concreteIndices(IndicesOptions indicesOptions, String... aliasesOrIndices) throws IndexMissingException, IllegalArgumentException { if (indicesOptions.expandWildcardsOpen() || indicesOptions.expandWildcardsClosed()) {