From bb15def36e87ab7fffb803bdd146a89f38b0f48f Mon Sep 17 00:00:00 2001 From: Clinton Gormley Date: Tue, 10 Jun 2014 17:35:49 +0200 Subject: [PATCH] Stats: Bugfixes and enhancements to indices stats API Bugs: * "groups" and "types" were being ignored * "completion_fields" as wildcards were not being resolved to fieldnames Enhancements: * Made "groups" and "types" support wildcards * Added missing tests Closes #6390 --- .../test/indices.stats/10_basic.yaml | 4 - .../test/indices.stats/10_index.yaml | 92 ++++++ .../test/indices.stats/11_metric.yaml | 110 +++++++ .../test/indices.stats/12_level.yaml | 69 +++++ .../test/indices.stats/13_fields.yaml | 271 ++++++++++++++++++ .../test/indices.stats/14_groups.yaml | 79 +++++ .../test/indices.stats/15_types.yaml | 82 ++++++ .../index/fielddata/ShardFieldData.java | 6 +- .../index/indexing/ShardIndexingService.java | 11 +- .../search/stats/ShardSearchService.java | 15 +- .../indices/stats/RestIndicesStatsAction.java | 17 +- .../AnalyzingCompletionLookupProvider.java | 14 +- .../indices/stats/SimpleIndexStatsTests.java | 215 +++++++++++++- 13 files changed, 938 insertions(+), 47 deletions(-) delete mode 100644 rest-api-spec/test/indices.stats/10_basic.yaml create mode 100644 rest-api-spec/test/indices.stats/10_index.yaml create mode 100644 rest-api-spec/test/indices.stats/11_metric.yaml create mode 100644 rest-api-spec/test/indices.stats/12_level.yaml create mode 100644 rest-api-spec/test/indices.stats/13_fields.yaml create mode 100644 rest-api-spec/test/indices.stats/14_groups.yaml create mode 100644 rest-api-spec/test/indices.stats/15_types.yaml diff --git a/rest-api-spec/test/indices.stats/10_basic.yaml b/rest-api-spec/test/indices.stats/10_basic.yaml deleted file mode 100644 index f197f38d1ce..00000000000 --- a/rest-api-spec/test/indices.stats/10_basic.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -"stats test": - - do: - indices.stats: {} diff --git a/rest-api-spec/test/indices.stats/10_index.yaml b/rest-api-spec/test/indices.stats/10_index.yaml new file mode 100644 index 00000000000..b07da61db0b --- /dev/null +++ b/rest-api-spec/test/indices.stats/10_index.yaml @@ -0,0 +1,92 @@ +--- + setup: + - do: + indices.create: + index: test1 + body: + settings: + number_of_shards: 5 + number_of_replicas: 1 + + - do: + indices.create: + index: test2 + body: + settings: + number_of_shards: 4 + number_of_replicas: 1 + - do: + index: + index: test1 + type: bar + id: 1 + body: { "foo": "bar" } + + - do: + index: + index: test2 + type: baz + id: 1 + body: { "foo": "baz" } + +--- +"Index - blank": + - do: + indices.stats: {} + + - match: { _shards.total: 18 } + - is_true: _all + - is_true: indices.test1 + - is_true: indices.test2 + +--- +"Index - all": + - do: + indices.stats: { index: _all } + + - match: { _shards.total: 18 } + - is_true: _all + - is_true: indices.test1 + - is_true: indices.test2 + + +--- +"Index - star": + - do: + indices.stats: { index: '*' } + + - match: { _shards.total: 18 } + - is_true: _all + - is_true: indices.test1 + - is_true: indices.test2 + +--- +"Index - one index": + - do: + indices.stats: { index: 'test1' } + + - match: { _shards.total: 10 } + - is_true: _all + - is_true: indices.test1 + - is_false: indices.test2 + +--- +"Index - multi-index": + - do: + indices.stats: { index: [test1, test2] } + + - match: { _shards.total: 18 } + - is_true: _all + - is_true: indices.test1 + - is_true: indices.test2 + +--- +"Index - pattern": + - do: + indices.stats: { index: '*2' } + + - match: { _shards.total: 8 } + - is_true: _all + - is_false: indices.test1 + - is_true: indices.test2 + diff --git a/rest-api-spec/test/indices.stats/11_metric.yaml b/rest-api-spec/test/indices.stats/11_metric.yaml new file mode 100644 index 00000000000..562d8f5f1cd --- /dev/null +++ b/rest-api-spec/test/indices.stats/11_metric.yaml @@ -0,0 +1,110 @@ +--- + setup: + + - do: + index: + index: test1 + type: bar + id: 1 + body: { "foo": "bar" } + + - do: + index: + index: test2 + type: baz + id: 1 + body: { "foo": "baz" } + +--- +"Metric - blank": + - do: + indices.stats: {} + + - is_true: _all.total.docs + - is_true: _all.total.store + - is_true: _all.total.indexing + - is_true: _all.total.get + - is_true: _all.total.search + - is_true: _all.total.merges + - is_true: _all.total.refresh + - is_true: _all.total.flush + - is_true: _all.total.warmer + - is_true: _all.total.filter_cache + - is_true: _all.total.id_cache + - is_true: _all.total.fielddata + - is_true: _all.total.percolate + - is_true: _all.total.completion + - is_true: _all.total.segments + - is_true: _all.total.translog + - is_true: _all.total.suggest + +--- +"Metric - _all": + - do: + indices.stats: { metric: _all } + + - is_true: _all.total.docs + - is_true: _all.total.store + - is_true: _all.total.indexing + - is_true: _all.total.get + - is_true: _all.total.search + - is_true: _all.total.merges + - is_true: _all.total.refresh + - is_true: _all.total.flush + - is_true: _all.total.warmer + - is_true: _all.total.filter_cache + - is_true: _all.total.id_cache + - is_true: _all.total.fielddata + - is_true: _all.total.percolate + - is_true: _all.total.completion + - is_true: _all.total.segments + - is_true: _all.total.translog + - is_true: _all.total.suggest + +--- +"Metric - one": + - do: + indices.stats: { metric: docs } + + - is_true: _all.total.docs + - is_false: _all.total.store + - is_false: _all.total.indexing + - is_false: _all.total.get + - is_false: _all.total.search + - is_false: _all.total.merges + - is_false: _all.total.refresh + - is_false: _all.total.flush + - is_false: _all.total.warmer + - is_false: _all.total.filter_cache + - is_false: _all.total.id_cache + - is_false: _all.total.fielddata + - is_false: _all.total.percolate + - is_false: _all.total.completion + - is_false: _all.total.segments + - is_false: _all.total.translog + - is_false: _all.total.suggest + +--- +"Metric - multi": + - do: + indices.stats: { metric: [ store, get, merge ] } + + - is_false: _all.total.docs + - is_true: _all.total.store + - is_false: _all.total.indexing + - is_true: _all.total.get + - is_false: _all.total.search + - is_true: _all.total.merges + - is_false: _all.total.refresh + - is_false: _all.total.flush + - is_false: _all.total.warmer + - is_false: _all.total.filter_cache + - is_false: _all.total.id_cache + - is_false: _all.total.fielddata + - is_false: _all.total.percolate + - is_false: _all.total.completion + - is_false: _all.total.segments + - is_false: _all.total.translog + - is_false: _all.total.suggest + + diff --git a/rest-api-spec/test/indices.stats/12_level.yaml b/rest-api-spec/test/indices.stats/12_level.yaml new file mode 100644 index 00000000000..11c1354eadc --- /dev/null +++ b/rest-api-spec/test/indices.stats/12_level.yaml @@ -0,0 +1,69 @@ +--- + setup: + + - do: + index: + index: test1 + type: bar + id: 1 + body: { "foo": "bar" } + + - do: + index: + index: test2 + type: baz + id: 1 + body: { "foo": "baz" } + +--- +"Level - blank": + - do: + indices.stats: {} + + - is_true: _all.total.docs + - is_true: _all.total.docs + - is_true: indices.test1.total.docs + - is_true: indices.test1.total.docs + - is_false: indices.test1.shards + - is_true: indices.test2.total.docs + - is_true: indices.test2.total.docs + - is_false: indices.test2.shards + +--- +"Level - indices": + - do: + indices.stats: { level: indices } + + - is_true: _all.total.docs + - is_true: _all.total.docs + - is_true: indices.test1.total.docs + - is_true: indices.test1.total.docs + - is_false: indices.test1.shards + - is_true: indices.test2.total.docs + - is_true: indices.test2.total.docs + - is_false: indices.test2.shards + +--- +"Level - cluster": + - do: + indices.stats: { level: cluster } + + - is_true: _all.total.docs + - is_true: _all.total.docs + - is_false: indices + + +--- +"Level - shards": + - do: + indices.stats: { level: shards } + + - is_true: _all.total.docs + - is_true: _all.total.docs + - is_true: indices.test1.total.docs + - is_true: indices.test1.total.docs + - is_true: indices.test1.shards.0.0.docs + - is_true: indices.test2.total.docs + - is_true: indices.test2.total.docs + - is_true: indices.test2.shards.0.0.docs + diff --git a/rest-api-spec/test/indices.stats/13_fields.yaml b/rest-api-spec/test/indices.stats/13_fields.yaml new file mode 100644 index 00000000000..5e96063d2ba --- /dev/null +++ b/rest-api-spec/test/indices.stats/13_fields.yaml @@ -0,0 +1,271 @@ +--- + setup: + + - do: + indices.create: + index: test1 + body: + mappings: + bar: + properties: + bar: + type: string + fields: + completion: + type: completion + baz: + type: string + fields: + completion: + type: completion + - do: + index: + index: test1 + type: bar + id: 1 + body: { "bar": "bar", "baz": "baz" } + + - do: + index: + index: test2 + type: baz + id: 1 + body: { "bar": "bar", "baz": "baz" } + + - do: + indices.refresh: {} + + - do: + search: + sort: bar,baz + +--- +"Fields - blank": + - do: + indices.stats: {} + + - gt: { _all.total.fielddata.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields + - gt: { _all.total.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields + +--- +"Fields - one": + - do: + indices.stats: { fields: bar } + + - gt: { _all.total.fielddata.memory_size_in_bytes: 0 } + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields.baz + - gt: { _all.total.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields.bar + +--- +"Fields - multi": + - do: + indices.stats: { fields: "bar,baz.completion" } + + - gt: { _all.total.fielddata.memory_size_in_bytes: 0 } + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields.baz + - gt: { _all.total.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields.bar\\.completion + - gt: { _all.total.completion.fields.baz\\.completion.size_in_bytes: 0 } + +--- +"Fields - star": + - do: + indices.stats: { fields: "*" } + + - gt: { _all.total.fielddata.memory_size_in_bytes: 0 } + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - gt: { _all.total.fielddata.fields.baz.memory_size_in_bytes: 0 } + - gt: { _all.total.completion.size_in_bytes: 0 } + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - gt: { _all.total.completion.fields.baz\\.completion.size_in_bytes: 0 } + +--- +"Fields - pattern": + - do: + indices.stats: { fields: "bar*" } + + - gt: { _all.total.fielddata.memory_size_in_bytes: 0 } + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields.baz + - gt: { _all.total.completion.size_in_bytes: 0 } + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields.baz\\.completion + +--- +"Fields - _all metric": + - do: + indices.stats: { fields: "bar*", metric: _all } + + - gt: { _all.total.fielddata.memory_size_in_bytes: 0 } + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields.baz + - gt: { _all.total.completion.size_in_bytes: 0 } + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields.baz\\.completion + +--- +"Fields - fielddata metric": + - do: + indices.stats: { fields: "bar*", metric: fielddata } + + - gt: { _all.total.fielddata.memory_size_in_bytes: 0 } + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields.baz + - is_false: _all.total.completion + +--- +"Fields - completion metric": + - do: + indices.stats: { fields: "bar*", metric: completion } + + - is_false: _all.total.fielddata + - gt: { _all.total.completion.size_in_bytes: 0 } + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields.baz\\.completion + +--- +"Fields - multi metric": + - do: + indices.stats: { fields: "bar*" , metric: [ completion, fielddata, search ]} + + - gt: { _all.total.fielddata.memory_size_in_bytes: 0 } + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields.baz + - gt: { _all.total.completion.size_in_bytes: 0 } + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields.baz\\.completion + +--- +"Fielddata fields - one": + - do: + indices.stats: { fielddata_fields: bar } + + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields.baz + - is_false: _all.total.completion.fields + +--- +"Fielddata fields - multi": + - do: + indices.stats: { fielddata_fields: "bar,baz,baz.completion" } + + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - gt: { _all.total.fielddata.fields.baz.memory_size_in_bytes: 0 } + - is_false: _all.total.completion.fields + +--- +"Fielddata fields - star": + - do: + indices.stats: { fielddata_fields: "*" } + + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - gt: { _all.total.fielddata.fields.baz.memory_size_in_bytes: 0 } + - is_false: _all.total.completion.fields + +--- +"Fielddata fields - pattern": + - do: + indices.stats: { fielddata_fields: "*r" } + + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields.baz + - is_false: _all.total.completion.fields + + +--- +"Fielddata fields - all metric": + - do: + indices.stats: { fielddata_fields: "*r", metric: _all } + + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields.baz + - is_false: _all.total.completion.fields + +--- +"Fielddata fields - one metric": + - do: + indices.stats: { fielddata_fields: "*r", metric: fielddata } + + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields.baz + - is_false: _all.total.completion.fields + + +--- +"Fielddata fields - multi metric": + - do: + indices.stats: { fielddata_fields: "*r", metric: [ fielddata, search] } + + - gt: { _all.total.fielddata.fields.bar.memory_size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields.baz + - is_false: _all.total.completion.fields + + +--- +"Completion fields - one": + - do: + indices.stats: { completion_fields: bar.completion } + + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields.baz\\.completion + - is_false: _all.total.fielddata.fields + +--- +"Completion fields - multi": + - do: + indices.stats: { completion_fields: "bar.completion,baz,baz.completion" } + + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - gt: { _all.total.completion.fields.baz\\.completion.size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields + +--- +"Completion fields - star": + - do: + indices.stats: { completion_fields: "*" } + + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - gt: { _all.total.completion.fields.baz\\.completion.size_in_bytes: 0 } + - is_false: _all.total.fielddata.fields + +--- +"Completion - pattern": + - do: + indices.stats: { completion_fields: "*r*" } + + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields.baz\\.completion + - is_false: _all.total.fielddata.fields + +--- +"Completion - all metric": + - do: + indices.stats: { completion_fields: "*r*", metric: _all } + + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields.baz\\.completion + - is_false: _all.total.fielddata.fields + +--- +"Completion - one metric": + - do: + indices.stats: { completion_fields: "*r*", metric: completion } + + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields.baz\\.completion + - is_false: _all.total.fielddata.fields + +--- +"Completion - multi metric": + - do: + indices.stats: { completion_fields: "*r*", metric: [ completion, search ] } + + - gt: { _all.total.completion.fields.bar\\.completion.size_in_bytes: 0 } + - is_false: _all.total.completion.fields.baz\\.completion + - is_false: _all.total.fielddata.fields + diff --git a/rest-api-spec/test/indices.stats/14_groups.yaml b/rest-api-spec/test/indices.stats/14_groups.yaml new file mode 100644 index 00000000000..9d0cab44d35 --- /dev/null +++ b/rest-api-spec/test/indices.stats/14_groups.yaml @@ -0,0 +1,79 @@ +--- + setup: + + - do: + index: + index: test1 + type: bar + id: 1 + body: { "bar": "bar", "baz": "baz" } + + - do: + search: + body: + stats: [ bar, baz ] + +--- +"Groups - blank": + - do: + indices.stats: {} + + - gt: { _all.total.search.query_total: 0 } + - is_false: _all.total.search.groups + +--- +"Groups - one": + - do: + indices.stats: { groups: bar } + + - gt: { _all.total.search.groups.bar.query_total: 0 } + - is_false: _all.total.search.groups.baz + +--- +"Groups - multi": + - do: + indices.stats: { groups: "bar,baz" } + + - gt: { _all.total.search.groups.bar.query_total: 0 } + - gt: { _all.total.search.groups.baz.query_total: 0 } + +--- +"Groups - star": + - do: + indices.stats: { groups: "*" } + + - gt: { _all.total.search.groups.bar.query_total: 0 } + - gt: { _all.total.search.groups.baz.query_total: 0 } + +--- +"Groups - pattern": + - do: + indices.stats: { groups: "*r" } + + - gt: { _all.total.search.groups.bar.query_total: 0 } + - is_false: _all.total.search.groups.baz + +--- +"Groups - _all metric": + - do: + indices.stats: { groups: bar, metric: _all } + + - gt: { _all.total.search.groups.bar.query_total: 0 } + - is_false: _all.total.search.groups.baz + +--- +"Groups - search metric": + - do: + indices.stats: { groups: bar, metric: search } + + - gt: { _all.total.search.groups.bar.query_total: 0 } + - is_false: _all.total.search.groups.baz + +--- +"Groups - multi metric": + - do: + indices.stats: { groups: bar, metric: [ indexing, search] } + + - gt: { _all.total.search.groups.bar.query_total: 0 } + - is_false: _all.total.search.groups.baz + diff --git a/rest-api-spec/test/indices.stats/15_types.yaml b/rest-api-spec/test/indices.stats/15_types.yaml new file mode 100644 index 00000000000..205f3bd35c8 --- /dev/null +++ b/rest-api-spec/test/indices.stats/15_types.yaml @@ -0,0 +1,82 @@ +--- + setup: + + - do: + index: + index: test1 + type: bar + id: 1 + body: { "bar": "bar", "baz": "baz" } + + - do: + index: + index: test2 + type: baz + id: 1 + body: { "bar": "bar", "baz": "baz" } + + +--- +"Types - blank": + - do: + indices.stats: {} + + - match: { _all.total.indexing.index_total: 2 } + - is_false: _all.total.indexing.types + +--- +"Types - one": + - do: + indices.stats: { types: bar } + + - match: { _all.total.indexing.types.bar.index_total: 1 } + - is_false: _all.total.indexing.types.baz + +--- +"Types - multi": + - do: + indices.stats: { types: "bar,baz" } + + - match: { _all.total.indexing.types.bar.index_total: 1 } + - match: { _all.total.indexing.types.baz.index_total: 1 } + +--- +"Types - star": + - do: + indices.stats: { types: "*" } + + - match: { _all.total.indexing.types.bar.index_total: 1 } + - match: { _all.total.indexing.types.baz.index_total: 1 } + +--- +"Types - pattern": + - do: + indices.stats: { types: "*r" } + + - match: { _all.total.indexing.types.bar.index_total: 1 } + - is_false: _all.total.indexing.types.baz + +--- +"Types - _all metric": + - do: + indices.stats: { types: bar, metric: _all } + + - match: { _all.total.indexing.types.bar.index_total: 1 } + - is_false: _all.total.indexing.types.baz + +--- +"Types - indexing metric": + - do: + indices.stats: { types: bar, metric: indexing } + + - match: { _all.total.indexing.types.bar.index_total: 1 } + - is_false: _all.total.indexing.types.baz + +--- +"Types - multi metric": + - do: + indices.stats: { types: bar, metric: [ indexing, search ] } + + - match: { _all.total.indexing.types.bar.index_total: 1 } + - is_false: _all.total.indexing.types.baz + diff --git a/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java index 35cbf32f4d3..a24962520fa 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java +++ b/src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java @@ -53,10 +53,8 @@ public class ShardFieldData extends AbstractIndexShardComponent implements Index if (fields != null && fields.length > 0) { fieldTotals = new ObjectLongOpenHashMap<>(); for (Map.Entry entry : perFieldTotals.entrySet()) { - for (String field : fields) { - if (Regex.simpleMatch(field, entry.getKey())) { - fieldTotals.put(entry.getKey(), entry.getValue().count()); - } + if (Regex.simpleMatch(fields, entry.getKey())) { + fieldTotals.put(entry.getKey(), entry.getValue().count()); } } } diff --git a/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java b/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java index 4378dcf8695..a20da023d1f 100644 --- a/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java +++ b/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.indexing.slowlog.ShardSlowLogIndexingService; @@ -63,17 +64,15 @@ public class ShardIndexingService extends AbstractIndexShardComponent { IndexingStats.Stats total = totalStats.stats(); Map typesSt = null; if (types != null && types.length > 0) { + typesSt = new HashMap<>(typesStats.size()); if (types.length == 1 && types[0].equals("_all")) { - typesSt = new HashMap<>(typesStats.size()); for (Map.Entry entry : typesStats.entrySet()) { typesSt.put(entry.getKey(), entry.getValue().stats()); } } else { - typesSt = new HashMap<>(types.length); - for (String type : types) { - StatsHolder statsHolder = typesStats.get(type); - if (statsHolder != null) { - typesSt.put(type, statsHolder.stats()); + for (Map.Entry entry : typesStats.entrySet()) { + if (Regex.simpleMatch(types, entry.getKey())) { + typesSt.put(entry.getKey(), entry.getValue().stats()); } } } diff --git a/src/main/java/org/elasticsearch/index/search/stats/ShardSearchService.java b/src/main/java/org/elasticsearch/index/search/stats/ShardSearchService.java index f3174a3bdad..f8c00a03cba 100644 --- a/src/main/java/org/elasticsearch/index/search/stats/ShardSearchService.java +++ b/src/main/java/org/elasticsearch/index/search/stats/ShardSearchService.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.search.slowlog.ShardSlowLogSearchService; import org.elasticsearch.index.settings.IndexSettings; @@ -61,17 +62,15 @@ public class ShardSearchService extends AbstractIndexShardComponent { SearchStats.Stats total = totalStats.stats(); Map groupsSt = null; if (groups != null && groups.length > 0) { + groupsSt = new HashMap<>(groupsStats.size()); if (groups.length == 1 && groups[0].equals("_all")) { - groupsSt = new HashMap<>(groupsStats.size()); for (Map.Entry entry : groupsStats.entrySet()) { groupsSt.put(entry.getKey(), entry.getValue().stats()); } } else { - groupsSt = new HashMap<>(groups.length); - for (String group : groups) { - StatsHolder statsHolder = groupsStats.get(group); - if (statsHolder != null) { - groupsSt.put(group, statsHolder.stats()); + for (Map.Entry entry : groupsStats.entrySet()) { + if (Regex.simpleMatch(groups, entry.getKey())) { + groupsSt.put(entry.getKey(), entry.getValue().stats()); } } } @@ -128,7 +127,6 @@ public class ShardSearchService extends AbstractIndexShardComponent { } } - public void onFetchPhase(SearchContext searchContext, long tookInNanos) { totalStats.fetchMetric.inc(tookInNanos); totalStats.fetchCurrent.dec(); @@ -187,8 +185,7 @@ public class ShardSearchService extends AbstractIndexShardComponent { public final CounterMetric fetchCurrent = new CounterMetric(); public SearchStats.Stats stats() { - return new SearchStats.Stats( - queryMetric.count(), TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()), queryCurrent.count(), + return new SearchStats.Stats(queryMetric.count(), TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()), queryCurrent.count(), fetchMetric.count(), TimeUnit.NANOSECONDS.toMillis(fetchMetric.sum()), fetchCurrent.count()); } diff --git a/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java b/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java index 1c8885db407..5bd8cbd6ee0 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestBuilderListener; -import java.io.IOException; import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -59,14 +58,6 @@ public class RestIndicesStatsAction extends BaseRestHandler { indicesStatsRequest.indices(Strings.splitStringByCommaToArray(request.param("index"))); indicesStatsRequest.types(Strings.splitStringByCommaToArray(request.param("types"))); - if (request.hasParam("groups")) { - indicesStatsRequest.groups(Strings.splitStringByCommaToArray(request.param("groups"))); - } - - if (request.hasParam("types")) { - indicesStatsRequest.types(Strings.splitStringByCommaToArray(request.param("types"))); - } - Set metrics = Strings.splitStringByCommaToSet(request.param("metric", "_all")); // short cut, if no metrics have been specified in URI if (metrics.size() == 1 && metrics.contains("_all")) { @@ -91,6 +82,14 @@ public class RestIndicesStatsAction extends BaseRestHandler { indicesStatsRequest.suggest(metrics.contains("suggest")); } + if (request.hasParam("groups")) { + indicesStatsRequest.groups(Strings.splitStringByCommaToArray(request.param("groups"))); + } + + if (request.hasParam("types")) { + indicesStatsRequest.types(Strings.splitStringByCommaToArray(request.param("types"))); + } + if (indicesStatsRequest.completion() && (request.hasParam("fields") || request.hasParam("completion_fields"))) { indicesStatsRequest.completionFields(request.paramAsStringArray("completion_fields", request.paramAsStringArray("fields", Strings.EMPTY_ARRAY))); } diff --git a/src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java b/src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java index f4cb591e4c5..f34cfe650fa 100644 --- a/src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java +++ b/src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java @@ -32,11 +32,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.fst.ByteSequenceOutputs; -import org.apache.lucene.util.fst.FST; -import org.apache.lucene.util.fst.PairOutputs; +import org.apache.lucene.util.fst.*; import org.apache.lucene.util.fst.PairOutputs.Pair; -import org.apache.lucene.util.fst.PositiveIntOutputs; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.CompletionLookupProvider; @@ -303,12 +300,9 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider if (fields == null || fields.length == 0) { continue; } - for (String field : fields) { - // support for getting fields by regex as in fielddata - if (Regex.simpleMatch(field, entry.getKey())) { - long fstSize = entry.getValue().fst.sizeInBytes(); - completionFields.addTo(field, fstSize); - } + if (Regex.simpleMatch(fields, entry.getKey())) { + long fstSize = entry.getValue().fst.sizeInBytes(); + completionFields.addTo(entry.getKey(), fstSize); } } diff --git a/src/test/java/org/elasticsearch/indices/stats/SimpleIndexStatsTests.java b/src/test/java/org/elasticsearch/indices/stats/SimpleIndexStatsTests.java index 72ffdddd72a..ba707657555 100644 --- a/src/test/java/org/elasticsearch/indices/stats/SimpleIndexStatsTests.java +++ b/src/test/java/org/elasticsearch/indices/stats/SimpleIndexStatsTests.java @@ -20,17 +20,16 @@ package org.elasticsearch.indices.stats; import org.apache.lucene.util.Version; -import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.admin.indices.stats.*; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; +import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; import org.junit.Test; import java.io.IOException; @@ -39,7 +38,6 @@ import java.util.Random; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; -import static org.elasticsearch.test.ElasticsearchIntegrationTest.*; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.*; @@ -323,6 +321,213 @@ public class SimpleIndexStatsTests extends ElasticsearchIntegrationTest { } } + @Test + public void testMultiIndex() throws Exception { + + createIndex("test1"); + createIndex("test2"); + + ensureGreen(); + + client().prepareIndex("test1", "type1", Integer.toString(1)).setSource("field", "value").execute().actionGet(); + client().prepareIndex("test1", "type2", Integer.toString(1)).setSource("field", "value").execute().actionGet(); + client().prepareIndex("test2", "type", Integer.toString(1)).setSource("field", "value").execute().actionGet(); + refresh(); + + int numShards1 = getNumShards("test1").totalNumShards; + int numShards2 = getNumShards("test2").totalNumShards; + + IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats(); + IndicesStatsResponse stats = builder.execute().actionGet(); + + assertThat(stats.getTotalShards(), equalTo(numShards1 + numShards2)); + + stats = builder.setIndices("_all").execute().actionGet(); + assertThat(stats.getTotalShards(), equalTo(numShards1 + numShards2)); + + stats = builder.setIndices("_all").execute().actionGet(); + assertThat(stats.getTotalShards(), equalTo(numShards1 + numShards2)); + + stats = builder.setIndices("*").execute().actionGet(); + assertThat(stats.getTotalShards(), equalTo(numShards1 + numShards2)); + + stats = builder.setIndices("test1").execute().actionGet(); + assertThat(stats.getTotalShards(), equalTo(numShards1)); + + stats = builder.setIndices("test1", "test2").execute().actionGet(); + assertThat(stats.getTotalShards(), equalTo(numShards1 + numShards2)); + + stats = builder.setIndices("*2").execute().actionGet(); + assertThat(stats.getTotalShards(), equalTo(numShards2)); + + } + + @Test + public void testFieldDataFieldsParam() throws Exception { + + createIndex("test1"); + + ensureGreen(); + + client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); + client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); + refresh(); + + client().prepareSearch("_all").addSort("bar", SortOrder.ASC).addSort("baz", SortOrder.ASC).execute().actionGet(); + + IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats(); + IndicesStatsResponse stats = builder.execute().actionGet(); + + assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0l)); + assertThat(stats.getTotal().fieldData.getFields(), is(nullValue())); + + stats = builder.setFieldDataFields("bar").execute().actionGet(); + assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0l)); + assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().lget(), greaterThan(0l)); + assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(false)); + + stats = builder.setFieldDataFields("bar", "baz").execute().actionGet(); + assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0l)); + assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().lget(), greaterThan(0l)); + assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().lget(), greaterThan(0l)); + + stats = builder.setFieldDataFields("*").execute().actionGet(); + assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0l)); + assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().lget(), greaterThan(0l)); + assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().lget(), greaterThan(0l)); + + stats = builder.setFieldDataFields("*r").execute().actionGet(); + assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0l)); + assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().lget(), greaterThan(0l)); + assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(false)); + + } + + @Test + public void testCompletionFieldsParam() throws Exception { + + assertAcked(prepareCreate("test1") + .addMapping( + "bar", + "{ \"properties\": { \"bar\": { \"type\": \"string\", \"fields\": { \"completion\": { \"type\": \"completion\" }}},\"baz\": { \"type\": \"string\", \"fields\": { \"completion\": { \"type\": \"completion\" }}}}}")); + ensureGreen(); + + client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); + client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); + refresh(); + + IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats(); + IndicesStatsResponse stats = builder.execute().actionGet(); + + assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0l)); + assertThat(stats.getTotal().completion.getFields(), is(nullValue())); + + stats = builder.setCompletionFields("bar.completion").execute().actionGet(); + assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0l)); + assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true)); + assertThat(stats.getTotal().completion.getFields().lget(), greaterThan(0l)); + assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(false)); + + stats = builder.setCompletionFields("bar.completion", "baz.completion").execute().actionGet(); + assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0l)); + assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true)); + assertThat(stats.getTotal().completion.getFields().lget(), greaterThan(0l)); + assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(true)); + assertThat(stats.getTotal().completion.getFields().lget(), greaterThan(0l)); + + stats = builder.setCompletionFields("*").execute().actionGet(); + assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0l)); + assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true)); + assertThat(stats.getTotal().completion.getFields().lget(), greaterThan(0l)); + assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(true)); + assertThat(stats.getTotal().completion.getFields().lget(), greaterThan(0l)); + + stats = builder.setCompletionFields("*r*").execute().actionGet(); + assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0l)); + assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true)); + assertThat(stats.getTotal().completion.getFields().lget(), greaterThan(0l)); + assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(false)); + + } + + @Test + public void testGroupsParam() throws Exception { + + createIndex("test1"); + + ensureGreen(); + + client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("foo","bar").execute().actionGet(); + refresh(); + + client().prepareSearch("_all").setStats("bar", "baz").execute().actionGet(); + + IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats(); + IndicesStatsResponse stats = builder.execute().actionGet(); + + assertThat(stats.getTotal().search.getTotal().getQueryCount(), greaterThan(0l)); + assertThat(stats.getTotal().search.getGroupStats(), is(nullValue())); + + stats = builder.setGroups("bar").execute().actionGet(); + assertThat(stats.getTotal().search.getGroupStats().get("bar").getQueryCount(), greaterThan(0l)); + assertThat(stats.getTotal().search.getGroupStats().containsKey("baz"), is(false)); + + stats = builder.setGroups("bar", "baz").execute().actionGet(); + assertThat(stats.getTotal().search.getGroupStats().get("bar").getQueryCount(), greaterThan(0l)); + assertThat(stats.getTotal().search.getGroupStats().get("baz").getQueryCount(), greaterThan(0l)); + + stats = builder.setGroups("*").execute().actionGet(); + assertThat(stats.getTotal().search.getGroupStats().get("bar").getQueryCount(), greaterThan(0l)); + assertThat(stats.getTotal().search.getGroupStats().get("baz").getQueryCount(), greaterThan(0l)); + + stats = builder.setGroups("*r").execute().actionGet(); + assertThat(stats.getTotal().search.getGroupStats().get("bar").getQueryCount(), greaterThan(0l)); + assertThat(stats.getTotal().search.getGroupStats().containsKey("baz"), is(false)); + + } + + @Test + public void testTypesParam() throws Exception { + + createIndex("test1"); + createIndex("test2"); + + ensureGreen(); + + client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("foo","bar").execute().actionGet(); + client().prepareIndex("test2", "baz", Integer.toString(1)).setSource("foo","bar").execute().actionGet(); + refresh(); + + IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats(); + IndicesStatsResponse stats = builder.execute().actionGet(); + + assertThat(stats.getTotal().indexing.getTotal().getIndexCount(), greaterThan(0l)); + assertThat(stats.getTotal().indexing.getTypeStats(), is(nullValue())); + + stats = builder.setTypes("bar").execute().actionGet(); + assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0l)); + assertThat(stats.getTotal().indexing.getTypeStats().containsKey("baz"), is(false)); + + stats = builder.setTypes("bar", "baz").execute().actionGet(); + assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0l)); + assertThat(stats.getTotal().indexing.getTypeStats().get("baz").getIndexCount(), greaterThan(0l)); + + stats = builder.setTypes("*").execute().actionGet(); + assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0l)); + assertThat(stats.getTotal().indexing.getTypeStats().get("baz").getIndexCount(), greaterThan(0l)); + + stats = builder.setTypes("*r").execute().actionGet(); + assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0l)); + assertThat(stats.getTotal().indexing.getTypeStats().containsKey("baz"), is(false)); + + } + private static void set(Flag flag, IndicesStatsRequestBuilder builder, boolean set) { switch (flag) { case Docs: