diff --git a/dev-tools/es_release_notes.pl b/dev-tools/es_release_notes.pl index b07a973d832..7e65731a411 100644 --- a/dev-tools/es_release_notes.pl +++ b/dev-tools/es_release_notes.pl @@ -33,6 +33,7 @@ my @Groups = qw( ); my %Group_Labels = ( breaking => 'Breaking changes', + build => 'Build', deprecation => 'Deprecations', doc => 'Docs', feature => 'New features', @@ -70,6 +71,14 @@ sub dump_issues { $month++; $year += 1900; + print <<"HTML"; + + + + + +HTML + for my $group ( @Groups, 'other' ) { my $group_issues = $issues->{$group} or next; print "

$Group_Labels{$group}

\n\n"; print "\n\n"; } + print "\n"; } #=================================== diff --git a/docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc index c2776b84797..e9c79ed59d8 100644 --- a/docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc +++ b/docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc @@ -30,10 +30,10 @@ MetricsAggregationBuilder aggregation = AggregationBuilders .scriptedMetric("agg") .initScript("_agg['heights'] = []") - .mapScript("if (doc['gender'].value == \"male\") " + + .mapScript(new Script("if (doc['gender'].value == \"male\") " + "{ _agg.heights.add(doc['height'].value) } " + "else " + - "{ _agg.heights.add(-1 * doc['height'].value) }"); + "{ _agg.heights.add(-1 * doc['height'].value) }")); -------------------------------------------------- You can also specify a `combine` script which will be executed on each shard: @@ -43,12 +43,12 @@ You can also specify a `combine` script which will be executed on each shard: MetricsAggregationBuilder aggregation = AggregationBuilders .scriptedMetric("agg") - .initScript("_agg['heights'] = []") - .mapScript("if (doc['gender'].value == \"male\") " + + .initScript(new Script("_agg['heights'] = []")) + .mapScript(new Script("if (doc['gender'].value == \"male\") " + "{ _agg.heights.add(doc['height'].value) } " + "else " + - "{ _agg.heights.add(-1 * doc['height'].value) }") - .combineScript("heights_sum = 0; for (t in _agg.heights) { heights_sum += t }; return heights_sum"); + "{ _agg.heights.add(-1 * doc['height'].value) }")) + .combineScript(new Script("heights_sum = 0; for (t in _agg.heights) { heights_sum += t }; return heights_sum")); -------------------------------------------------- You can also specify a `reduce` script which will be executed on the node which gets the request: @@ -58,13 +58,13 @@ You can also specify a `reduce` script which will be executed on the node which MetricsAggregationBuilder aggregation = AggregationBuilders .scriptedMetric("agg") - .initScript("_agg['heights'] = []") - .mapScript("if (doc['gender'].value == \"male\") " + + .initScript(new Script("_agg['heights'] = []")) + .mapScript(new Script("if (doc['gender'].value == \"male\") " + "{ _agg.heights.add(doc['height'].value) } " + "else " + - "{ _agg.heights.add(-1 * doc['height'].value) }") - .combineScript("heights_sum = 0; for (t in _agg.heights) { heights_sum += t }; return heights_sum") - .reduceScript("heights_sum = 0; for (a in _aggs) { heights_sum += a }; return heights_sum"); + "{ _agg.heights.add(-1 * doc['height'].value) }")) + .combineScript(new Script("heights_sum = 0; for (t in _agg.heights) { heights_sum += t }; return heights_sum")) + .reduceScript(new Script("heights_sum = 0; for (a in _aggs) { heights_sum += a }; return heights_sum")); -------------------------------------------------- diff --git a/docs/java-api/update.asciidoc b/docs/java-api/update.asciidoc index 2de835755c6..ea25ec0c2d2 100644 --- a/docs/java-api/update.asciidoc +++ b/docs/java-api/update.asciidoc @@ -22,7 +22,7 @@ Or you can use `prepareUpdate()` method: [source,java] -------------------------------------------------- client.prepareUpdate("ttl", "doc", "1") - .setScript("ctx._source.gender = \"male\"" <1> , ScriptService.ScriptType.INLINE) + .setScript(new Script("ctx._source.gender = \"male\"" <1> , ScriptService.ScriptType.INLINE, null, null)) .get(); client.prepareUpdate("ttl", "doc", "1") @@ -46,7 +46,7 @@ The update API allows to update a document based on a script provided: [source,java] -------------------------------------------------- UpdateRequest updateRequest = new UpdateRequest("ttl", "doc", "1") - .script("ctx._source.gender = \"male\""); + .script(new Script("ctx._source.gender = \"male\"")); client.update(updateRequest).get(); -------------------------------------------------- diff --git a/docs/reference/aggregations.asciidoc b/docs/reference/aggregations.asciidoc index 2464873b452..b1b34ee66df 100644 --- a/docs/reference/aggregations.asciidoc +++ b/docs/reference/aggregations.asciidoc @@ -73,8 +73,6 @@ Some aggregations work on values extracted from the aggregated documents. Typica a specific document field which is set using the `field` key for the aggregations. It is also possible to define a <> which will generate the values (per document). -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. - When both `field` and `script` settings are configured for the aggregation, the script will be treated as a `value script`. While normal scripts are evaluated on a document level (i.e. the script has access to all the data associated with the document), value scripts are evaluated on the *value* level. In this mode, the values are extracted diff --git a/docs/reference/aggregations/bucket/range-aggregation.asciidoc b/docs/reference/aggregations/bucket/range-aggregation.asciidoc index f7bfcab0644..d428d44523f 100644 --- a/docs/reference/aggregations/bucket/range-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/range-aggregation.asciidoc @@ -128,8 +128,6 @@ It is also possible to customize the key for each range: ==== Script -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. - [source,js] -------------------------------------------------- { @@ -148,6 +146,33 @@ TIP: The `script` parameter expects an inline script. Use `script_id` for indexe } -------------------------------------------------- +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + "aggs" : { + "price_ranges" : { + "range" : { + "script" : { + "file": "my_script", + "params": { + "field": "price" + } + }, + "ranges" : [ + { "to" : 50 }, + { "from" : 50, "to" : 100 }, + { "from" : 100 } + ] + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. + ==== Value Script Lets say the product prices are in USD but we would like to get the price ranges in EURO. We can use value script to convert the prices prior the aggregation (assuming conversion rate of 0.8) diff --git a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc index 1e329db1df4..80c747e61a5 100644 --- a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc @@ -358,13 +358,6 @@ Customized scores can be implemented via a script: -------------------------------------------------- Scripts can be inline (as in above example), indexed or stored on disk. For details on the options, see <>. -Parameters need to be set as follows: - -[horizontal] -`script`:: Inline script, name of script file or name of indexed script. Mandatory. -`script_type`:: One of "inline" (default), "indexed" or "file". -`lang`:: Script language (default "groovy") -`params`:: Script parameters (default empty). Available parameters in the script are diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc index cf401126c8a..70bdb00d184 100644 --- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc @@ -441,7 +441,27 @@ Generating the terms using a script: } -------------------------------------------------- -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + "aggs" : { + "genders" : { + "terms" : { + "script" : { + "file": "my_script", + "params": { + "field": "gender" + } + } + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. ==== Value Script diff --git a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc index 8e0d2b4b5e7..f81cd3eee33 100644 --- a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc @@ -47,7 +47,29 @@ Computing the average grade based on a script: } -------------------------------------------------- -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + ..., + + "aggs" : { + "avg_grade" : { + "avg" : { + "script" : { + "file": "my_script", + "params": { + "field": "grade" + } + } + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. ===== Value Script @@ -63,9 +85,11 @@ It turned out that the exam was way above the level of the students and a grade "avg_corrected_grade" : { "avg" : { "field" : "grade", - "script" : "_value * correction", - "params" : { - "correction" : 1.2 + "script" : { + "inline": "_value * correction", + "params" : { + "correction" : 1.2 + } } } } diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc index 8e34e16f7a8..0b484288b1c 100644 --- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc @@ -153,7 +153,28 @@ however since hashes need to be computed on the fly. } -------------------------------------------------- -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + "aggs" : { + "author_count" : { + "cardinality" : { + "script" : { + "file": "my_script", + "params": { + "first_name_field": "author.first_name", + "last_name_field": "author.last_name" + } + } + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. ==== Missing value diff --git a/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc b/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc index 0f65b7670cf..30a5acf6809 100644 --- a/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc @@ -91,7 +91,29 @@ Computing the grades stats based on a script: } -------------------------------------------------- -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + ..., + + "aggs" : { + "grades_stats" : { + "extended_stats" : { + "script" : { + "file": "my_script", + "params": { + "field": "grade" + } + } + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. ===== Value Script @@ -107,9 +129,11 @@ It turned out that the exam was way above the level of the students and a grade "grades_stats" : { "extended_stats" : { "field" : "grade", - "script" : "_value * correction", - "params" : { - "correction" : 1.2 + "script" : { + "inline": "_value * correction", + "params" : { + "correction" : 1.2 + } } } } diff --git a/docs/reference/aggregations/metrics/max-aggregation.asciidoc b/docs/reference/aggregations/metrics/max-aggregation.asciidoc index 856adc4b03d..2a641fda5dc 100644 --- a/docs/reference/aggregations/metrics/max-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/max-aggregation.asciidoc @@ -44,7 +44,27 @@ Computing the max price value across all document, this time using a script: } -------------------------------------------------- -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + "aggs" : { + "max_price" : { + "max" : { + "script" : { + "file": "my_script", + "params": { + "field": "price" + } + } + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. ==== Value Script @@ -57,9 +77,11 @@ Let's say that the prices of the documents in our index are in USD, but we would "max_price_in_euros" : { "max" : { "field" : "price", - "script" : "_value * conversion_rate", - "params" : { - "conversion_rate" : 1.2 + "script" : { + "inline": "_value * conversion_rate", + "params" : { + "conversion_rate" : 1.2 + } } } } diff --git a/docs/reference/aggregations/metrics/min-aggregation.asciidoc b/docs/reference/aggregations/metrics/min-aggregation.asciidoc index c7424d5570b..7698a41202c 100644 --- a/docs/reference/aggregations/metrics/min-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/min-aggregation.asciidoc @@ -44,7 +44,27 @@ Computing the min price value across all document, this time using a script: } -------------------------------------------------- -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + "aggs" : { + "min_price" : { + "min" : { + "script" : { + "file": "my_script", + "params": { + "field": "price" + } + } + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. ==== Value Script @@ -57,9 +77,11 @@ Let's say that the prices of the documents in our index are in USD, but we would "min_price_in_euros" : { "min" : { "field" : "price", - "script" : "_value * conversion_rate", - "params" : { - "conversion_rate" : 1.2 + "script" : + "inline": "_value * conversion_rate", + "params" : { + "conversion_rate" : 1.2 + } } } } diff --git a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc index d5262beb6ef..ecad363886d 100644 --- a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc @@ -100,9 +100,11 @@ a script to convert them on-the-fly: "aggs" : { "load_time_outlier" : { "percentiles" : { - "script" : "doc['load_time'].value / timeUnit", <1> - "params" : { - "timeUnit" : 1000 <2> + "script" : { + "inline": "doc['load_time'].value / timeUnit", <1> + "params" : { + "timeUnit" : 1000 <2> + } } } } @@ -113,7 +115,27 @@ a script to convert them on-the-fly: script to generate values which percentiles are calculated on <2> Scripting supports parameterized input just like any other script -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + "aggs" : { + "load_time_outlier" : { + "percentiles" : { + "script" : { + "file": "my_script", + "params" : { + "timeUnit" : 1000 + } + } + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. [[search-aggregations-metrics-percentile-aggregation-approximation]] ==== Percentiles are (usually) approximate diff --git a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc index a494a0a5d00..5da59061e0b 100644 --- a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc @@ -72,9 +72,11 @@ a script to convert them on-the-fly: "load_time_outlier" : { "percentile_ranks" : { "values" : [3, 5], - "script" : "doc['load_time'].value / timeUnit", <1> - "params" : { - "timeUnit" : 1000 <2> + "script" : { + "inline": "doc['load_time'].value / timeUnit", <1> + "params" : { + "timeUnit" : 1000 <2> + } } } } @@ -85,7 +87,28 @@ a script to convert them on-the-fly: script to generate values which percentile ranks are calculated on <2> Scripting supports parameterized input just like any other script -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + "aggs" : { + "load_time_outlier" : { + "percentile_ranks" : { + "values" : [3, 5], + "script" : { + "file": "my_script", + "params" : { + "timeUnit" : 1000 + } + } + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. ==== Missing value @@ -108,3 +131,4 @@ had a value. -------------------------------------------------- <1> Documents without a value in the `grade` field will fall into the same bucket as documents that have the value `10`. + diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc index a775d545409..6db8c82a9e8 100644 --- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc @@ -45,6 +45,42 @@ The response for the above aggregation: } -------------------------------------------------- +The above example can also be specified using file scripts as follows: + +[source,js] +-------------------------------------------------- +{ + "query" : { + "match_all" : {} + }, + "aggs": { + "profit": { + "scripted_metric": { + "init_script" : { + "file": "my_init_script" + }, + "map_script" : { + "file": "my_map_script" + }, + "combine_script" : { + "file": "my_combine_script" + }, + "params": { + "field": "amount" <1> + }, + "reduce_script" : { + "file": "my_reduce_script" + }, + } + } + } +} +-------------------------------------------------- + +<1> script parameters for init, map and combine scripts must be specified in a global `params` object so that it can be share between the scripts + +For more details on specifying scripts see <>. + ==== Scope of scripts The scripted metric aggregation uses scripts at 4 stages of its execution: @@ -225,13 +261,4 @@ params:: Optional. An object whose contents will be passed as variable -------------------------------------------------- reduce_params:: Optional. An object whose contents will be passed as variables to the `reduce_script`. This can be useful to allow the user to control the behavior of the reduce phase. If this is not specified the variable will be undefined in the reduce_script execution. -lang:: Optional. The script language used for the scripts. If this is not specified the default scripting language is used. -init_script_file:: Optional. Can be used in place of the `init_script` parameter to provide the script using in a file. -init_script_id:: Optional. Can be used in place of the `init_script` parameter to provide the script using an indexed script. -map_script_file:: Optional. Can be used in place of the `map_script` parameter to provide the script using in a file. -map_script_id:: Optional. Can be used in place of the `map_script` parameter to provide the script using an indexed script. -combine_script_file:: Optional. Can be used in place of the `combine_script` parameter to provide the script using in a file. -combine_script_id:: Optional. Can be used in place of the `combine_script` parameter to provide the script using an indexed script. -reduce_script_file:: Optional. Can be used in place of the `reduce_script` parameter to provide the script using in a file. -reduce_script_id:: Optional. Can be used in place of the `reduce_script` parameter to provide the script using an indexed script. diff --git a/docs/reference/aggregations/metrics/stats-aggregation.asciidoc b/docs/reference/aggregations/metrics/stats-aggregation.asciidoc index 429be4b8c4d..852c1c3f7a9 100644 --- a/docs/reference/aggregations/metrics/stats-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/stats-aggregation.asciidoc @@ -53,7 +53,29 @@ Computing the grades stats based on a script: } -------------------------------------------------- -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + ..., + + "aggs" : { + "grades_stats" : { + "stats" : { + "script" : { + "file": "my_script", + "params" : { + "field" : "grade" + } + } + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. ===== Value Script @@ -69,9 +91,11 @@ It turned out that the exam was way above the level of the students and a grade "grades_stats" : { "stats" : { "field" : "grade", - "script" : "_value * correction", - "params" : { - "correction" : 1.2 + "script" : + "inline": "_value * correction", + "params" : { + "correction" : 1.2 + } } } } diff --git a/docs/reference/aggregations/metrics/sum-aggregation.asciidoc b/docs/reference/aggregations/metrics/sum-aggregation.asciidoc index 2d16129d15f..98286e9396f 100644 --- a/docs/reference/aggregations/metrics/sum-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/sum-aggregation.asciidoc @@ -55,7 +55,29 @@ Computing the intraday return based on a script: } -------------------------------------------------- -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + ..., + + "aggs" : { + "intraday_return" : { + "sum" : { + "script" : { + "file": "my_script", + "params" : { + "field" : "change" + } + } + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. ===== Value Script @@ -71,7 +93,8 @@ Computing the sum of squares over all stock tick changes: "daytime_return" : { "sum" : { "field" : "change", - "script" : "_value * _value" } + "script" : "_value * _value" + } } } } diff --git a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc index ed5e23ee339..fa2bfdbbb9d 100644 --- a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc @@ -48,4 +48,26 @@ Counting the values generated by a script: } -------------------------------------------------- -TIP: The `script` parameter expects an inline script. Use `script_id` for indexed scripts and `script_file` for scripts in the `config/scripts/` directory. +This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax: + +[source,js] +-------------------------------------------------- +{ + ..., + + "aggs" : { + "grades_count" : { + "value_count" : { + "script" : { + "file": "my_script", + "params" : { + "field" : "grade" + } + } + } + } + } +} +-------------------------------------------------- + +TIP: for indexed scripts replace the `file` parameter with an `id` parameter. diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc index 2760a125ff8..aaf12079747 100644 --- a/docs/reference/docs/bulk.asciidoc +++ b/docs/reference/docs/bulk.asciidoc @@ -187,7 +187,7 @@ the options. Curl example with update actions: { "update" : {"_id" : "1", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} } { "doc" : {"field" : "value"} } { "update" : { "_id" : "0", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} } -{ "script" : "ctx._source.counter += param1", "lang" : "js", "params" : {"param1" : 1}, "upsert" : {"counter" : 1}} +{ "script" : { "inline": "ctx._source.counter += param1", "lang" : "js", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}} { "update" : {"_id" : "2", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} } { "doc" : {"field" : "value"}, "doc_as_upsert" : true } -------------------------------------------------- diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index 4236e06754a..485b31cb037 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -28,9 +28,11 @@ Now, we can execute a script that would increment the counter: [source,js] -------------------------------------------------- curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{ - "script" : "ctx._source.counter += count", - "params" : { - "count" : 4 + "script" : { + "inline": "ctx._source.counter += count", + "params" : { + "count" : 4 + } } }' -------------------------------------------------- @@ -41,9 +43,11 @@ will still add it, since its a list): [source,js] -------------------------------------------------- curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{ - "script" : "ctx._source.tags += tag", - "params" : { - "tag" : "blue" + "script" : { + "inline": "ctx._source.tags += tag", + "params" : { + "tag" : "blue" + } } }' -------------------------------------------------- @@ -71,9 +75,11 @@ And, we can delete the doc if the tags contain blue, or ignore (noop): [source,js] -------------------------------------------------- curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{ - "script" : "ctx._source.tags.contains(tag) ? ctx.op = \"delete\" : ctx.op = \"none\"", - "params" : { - "tag" : "blue" + "script" : { + "inline": "ctx._source.tags.contains(tag) ? ctx.op = \"delete\" : ctx.op = \"none\"", + "params" : { + "tag" : "blue" + } } }' -------------------------------------------------- @@ -136,9 +142,11 @@ index the fresh doc: [source,js] -------------------------------------------------- curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{ - "script" : "ctx._source.counter += count", - "params" : { - "count" : 4 + "script" : { + "inline": "ctx._source.counter += count", + "params" : { + "count" : 4 + } }, "upsert" : { "counter" : 1 @@ -153,13 +161,15 @@ new `scripted_upsert` parameter with the value `true`. [source,js] -------------------------------------------------- curl -XPOST 'localhost:9200/sessions/session/dh3sgudg8gsrgl/_update' -d '{ - "script_id" : "my_web_session_summariser", "scripted_upsert":true, - "params" : { - "pageViewEvent" : { - "url":"foo.com/bar", - "response":404, - "time":"2014-01-01 12:32" + "script" : { + "id": "my_web_session_summariser", + "params" : { + "pageViewEvent" : { + "url":"foo.com/bar", + "response":404, + "time":"2014-01-01 12:32" + } } }, "upsert" : { diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index f21ed1162e1..d897b97ab27 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -566,7 +566,7 @@ Which means that we just successfully bulk indexed 1000 documents into the bank === The Search API -Now let's start with some simple searches. There are two basic ways to run searches: one is by sending search parameters through the <> and the other by sending them through the<>. The request body method allows you to be more expressive and also to define your searches in a more readable JSON format. We'll try one example of the request URI method but for the remainder of this tutorial, we will exclusively be using the request body method. +Now let's start with some simple searches. There are two basic ways to run searches: one is by sending search parameters through the <> and the other by sending them through the <>. The request body method allows you to be more expressive and also to define your searches in a more readable JSON format. We'll try one example of the request URI method but for the remainder of this tutorial, we will exclusively be using the request body method. The REST API for search is accessible from the `_search` endpoint. This example returns all documents in the bank index: diff --git a/docs/reference/index-modules/fielddata.asciidoc b/docs/reference/index-modules/fielddata.asciidoc index 68764a05a27..5698af6d4da 100644 --- a/docs/reference/index-modules/fielddata.asciidoc +++ b/docs/reference/index-modules/fielddata.asciidoc @@ -149,6 +149,7 @@ field data format. Computes and stores field data data-structures on disk at indexing time. [float] +[[global-ordinals]] ==== Global ordinals Global ordinals is a data-structure on top of field data, that maintains an @@ -182,6 +183,7 @@ ordinals is a small because it is very efficiently compressed. Eager loading of can move the loading time from the first search request, to the refresh itself. [float] +[[fielddata-loading]] === Fielddata loading By default, field data is loaded lazily, ie. the first time that a query that diff --git a/docs/reference/mapping/fields/parent-field.asciidoc b/docs/reference/mapping/fields/parent-field.asciidoc index 260ab57af98..8caec1d4a71 100644 --- a/docs/reference/mapping/fields/parent-field.asciidoc +++ b/docs/reference/mapping/fields/parent-field.asciidoc @@ -1,6 +1,9 @@ [[mapping-parent-field]] === `_parent` +TIP: It is highly recommend to reindex all indices with `_parent` field created before version 2.x. + The reason for this is to gain from all the optimizations added with the 2.0 release. + The parent field mapping is defined on a child mapping, and points to the parent type this child relates to. For example, in case of a `blog` type and a `blog_tag` type child document, the mapping for `blog_tag` @@ -20,8 +23,34 @@ should be: The mapping is automatically stored and indexed (meaning it can be searched on using the `_parent` field notation). -==== Field data loading +==== Limitations -Contrary to other fields the fielddata loading is not `lazy`, but `eager`. The reason for this is that when this -field has been enabled it is going to be used in parent/child queries, which heavily relies on field data to perform -efficiently. This can already be observed during indexing after refresh either automatically or manually has been executed. +The `_parent.type` setting can only point to a type that doesn't exist yet. +This means that a type can't become a parent type after is has been created. + +The `parent.type` setting can't point to itself. This means self referential +parent/child isn't supported. + +Parent/child queries (`has_child` & `has_parent`) can't be used in index aliases. + +==== Global ordinals + +Parent-child uses <> to speed up joins and global ordinals need to be rebuilt after any change to a shard. +The more parent id values are stored in a shard, the longer it takes to rebuild global ordinals for the `_parent` field. + +Global ordinals, by default, are built lazily: the first parent-child query or aggregation after a refresh will trigger building of global ordinals. +This can introduce a significant latency spike for your users. You can use <> to shift the cost of building global ordinals +from query time to refresh time, by mapping the _parent field as follows: + +==== Memory usage + +The only on heap memory used by parent/child is the global ordinals for the `_parent` field. + +How much memory is used for the global ordianls for the `_parent` field in the fielddata cache +can be checked via the <> or <> +APIS, eg: + +[source,js] +-------------------------------------------------- +curl -XGET "http://localhost:9200/_stats/fielddata?pretty&human&fielddata_fields=_parent" +-------------------------------------------------- diff --git a/docs/reference/mapping/transform.asciidoc b/docs/reference/mapping/transform.asciidoc index 5235afcfd96..9377336518a 100644 --- a/docs/reference/mapping/transform.asciidoc +++ b/docs/reference/mapping/transform.asciidoc @@ -10,11 +10,13 @@ field. Example: { "example" : { "transform" : { - "script" : "if (ctx._source['title']?.startsWith('t')) ctx._source['suggest'] = ctx._source['content']", - "params" : { - "variable" : "not used but an example anyway" - }, - "lang": "groovy" + "script" : { + "inline": "if (ctx._source['title']?.startsWith('t')) ctx._source['suggest'] = ctx._source['content']", + "params" : { + "variable" : "not used but an example anyway" + }, + "lang": "groovy" + } }, "properties": { "title": { "type": "string" }, diff --git a/docs/reference/migration/migrate_2_0.asciidoc b/docs/reference/migration/migrate_2_0.asciidoc index 34b022b95d2..d45242b573f 100644 --- a/docs/reference/migration/migrate_2_0.asciidoc +++ b/docs/reference/migration/migrate_2_0.asciidoc @@ -626,3 +626,19 @@ anymore, it will only highlight fields that were queried. The `match` query with type set to `match_phrase_prefix` is not supported by the postings highlighter. No highlighted snippets will be returned. +[float] +=== Parent/child + +Parent/child has been rewritten completely to reduce memory usage and to execute +`has_child` and `has_parent` queries faster and more efficient. The `_parent` field +uses doc values by default. The refactored and improved implementation is only active +for indices created on or after version 2.0. + +In order to benefit for all performance and memory improvements we recommend to reindex all +indices that have the `_parent` field created before was upgraded to 2.0. + +The following breaks in backwards compatability have been made on indices with the `_parent` field +created on or after clusters with version 2.0: +* The `type` option on the `_parent` field can only point to a parent type that doesn't exist yet, + so this means that an existing type/mapping can no longer become a parent type. +* The `has_child` and `has_parent` queries can no longer be use in alias filters. \ No newline at end of file diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc index 750802c4ec2..0550542b4a2 100644 --- a/docs/reference/modules/scripting.asciidoc +++ b/docs/reference/modules/scripting.asciidoc @@ -29,7 +29,7 @@ GET /_search { "script_fields": { "my_field": { - "script": "1 + my_var", + "inline": "1 + my_var", "params": { "my_var": 2 } @@ -38,7 +38,7 @@ GET /_search } ----------------------------------- -Save the contents of the script as a file called `config/scripts/my_script.groovy` +Save the contents of the `inline` field as a file called `config/scripts/my_script.groovy` on every data node in the cluster: [source,js] @@ -54,7 +54,7 @@ GET /_search { "script_fields": { "my_field": { - "script_file": "my_script", + "file": "my_script", "params": { "my_var": 2 } @@ -67,9 +67,9 @@ GET /_search Additional `lang` plugins are provided to allow to execute scripts in -different languages. All places where a `script` parameter can be used, a `lang` parameter -(on the same level) can be provided to define the language of the -script. The following are the supported scripting languages: +different languages. All places where a script can be used, a `lang` parameter +can be provided to define the language of the script. The following are the +supported scripting languages: [cols="<,<,<",options="header",] |======================================================================= @@ -120,7 +120,7 @@ curl -XPOST localhost:9200/_search -d '{ { "script_score": { "lang": "groovy", - "script_file": "calculate-score", + "file": "calculate-score", "params": { "my_modifier": 8 } @@ -162,8 +162,8 @@ curl -XPOST localhost:9200/_scripts/groovy/indexedCalculateScore -d '{ This will create a document with id: `indexedCalculateScore` and type: `groovy` in the `.scripts` index. The type of the document is the language used by the script. -This script can be accessed at query time by appending `_id` to -the script parameter and passing the script id. So `script` becomes `script_id`.: +This script can be accessed at query time by using the `id` script parameter and passing +the script id: [source,js] -------------------------------------------------- @@ -178,7 +178,7 @@ curl -XPOST localhost:9200/_search -d '{ "functions": [ { "script_score": { - "script_id": "indexedCalculateScore", + "id": "indexedCalculateScore", "lang" : "groovy", "params": { "my_modifier": 8 diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc index 4588b4c7858..a5618a23c48 100644 --- a/docs/reference/query-dsl/function-score-query.asciidoc +++ b/docs/reference/query-dsl/function-score-query.asciidoc @@ -120,12 +120,14 @@ script, and provide parameters to it: [source,js] -------------------------------------------------- "script_score": { - "lang": "lang", - "params": { - "param1": value1, - "param2": value2 - }, - "script": "_score * doc['my_numeric_field'].value / pow(param1, param2)" + "script": { + "lang": "lang", + "params": { + "param1": value1, + "param2": value2 + }, + "inline": "_score * doc['my_numeric_field'].value / pow(param1, param2)" + } } -------------------------------------------------- diff --git a/docs/reference/query-dsl/has-child-query.asciidoc b/docs/reference/query-dsl/has-child-query.asciidoc index e974f58001e..606c6a58459 100644 --- a/docs/reference/query-dsl/has-child-query.asciidoc +++ b/docs/reference/query-dsl/has-child-query.asciidoc @@ -72,21 +72,3 @@ a match: The `min_children` and `max_children` parameters can be combined with the `score_mode` parameter. - -[float] -=== Memory Considerations - -In order to support parent-child joins, all of the (string) parent IDs -must be resident in memory (in the <>. -Additionally, every child document is mapped to its parent using a long -value (approximately). It is advisable to keep the string parent ID short -in order to reduce memory usage. - -You can check how much memory is being used by the `_parent` field in the fielddata cache -using the <> or <> -APIS, eg: - -[source,js] --------------------------------------------------- -curl -XGET "http://localhost:9200/_stats/fielddata?pretty&human&fielddata_fields=_parent" --------------------------------------------------- diff --git a/docs/reference/query-dsl/has-parent-query.asciidoc b/docs/reference/query-dsl/has-parent-query.asciidoc index f0ab59a1b1e..cbd4fb9358e 100644 --- a/docs/reference/query-dsl/has-parent-query.asciidoc +++ b/docs/reference/query-dsl/has-parent-query.asciidoc @@ -47,23 +47,3 @@ matching parent document. The score type can be specified with the } } -------------------------------------------------- - -[float] -=== Memory Considerations - -In order to support parent-child joins, all of the (string) parent IDs -must be resident in memory (in the <>. -Additionally, every child document is mapped to its parent using a long -value (approximately). It is advisable to keep the string parent ID short -in order to reduce memory usage. - -You can check how much memory is being used by the `_parent` field in the fielddata cache -using the <> or <> -APIS, eg: - -[source,js] --------------------------------------------------- -curl -XGET "http://localhost:9200/_stats/fielddata?pretty&human&fielddata_fields=_parent" --------------------------------------------------- - - diff --git a/docs/reference/query-dsl/script-query.asciidoc b/docs/reference/query-dsl/script-query.asciidoc index 899f176578e..4c307f2556f 100644 --- a/docs/reference/query-dsl/script-query.asciidoc +++ b/docs/reference/query-dsl/script-query.asciidoc @@ -34,9 +34,11 @@ to use the ability to pass parameters to the script itself, for example: }, "filter" : { "script" : { - "script" : "doc['num1'].value > param1" - "params" : { - "param1" : 5 + "script" : { + "inline" : "doc['num1'].value > param1" + "params" : { + "param1" : 5 + } } } } diff --git a/docs/reference/query-dsl/template-query.asciidoc b/docs/reference/query-dsl/template-query.asciidoc index 5d68992ff54..31728fe9993 100644 --- a/docs/reference/query-dsl/template-query.asciidoc +++ b/docs/reference/query-dsl/template-query.asciidoc @@ -12,7 +12,7 @@ GET /_search { "query": { "template": { - "query": { "match": { "text": "{{query_string}}" }}, + "inline": { "match": { "text": "{{query_string}}" }}, "params" : { "query_string" : "all about search" } @@ -45,7 +45,7 @@ GET /_search { "query": { "template": { - "query": "{ \"match\": { \"text\": \"{{query_string}}\" }}", <1> + "inline": "{ \"match\": { \"text\": \"{{query_string}}\" }}", <1> "params" : { "query_string" : "all about search" } diff --git a/docs/reference/search/request/script-fields.asciidoc b/docs/reference/search/request/script-fields.asciidoc index 46b169838a0..596aba31d82 100644 --- a/docs/reference/search/request/script-fields.asciidoc +++ b/docs/reference/search/request/script-fields.asciidoc @@ -15,9 +15,11 @@ evaluation>> (based on different fields) for each hit, for example: "script" : "doc['my_field_name'].value * 2" }, "test2" : { - "script" : "doc['my_field_name'].value * factor", - "params" : { - "factor" : 2.0 + "script" : { + "inline": "doc['my_field_name'].value * factor", + "params" : { + "factor" : 2.0 + } } } } diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc index 1e4218bb61d..58f42d8fdd8 100644 --- a/docs/reference/search/request/sort.asciidoc +++ b/docs/reference/search/request/sort.asciidoc @@ -318,10 +318,12 @@ Allow to sort based on custom scripts, here is an example: }, "sort" : { "_script" : { - "script" : "doc['field_name'].value * factor", "type" : "number", - "params" : { - "factor" : 1.1 + "script" : { + "inline": "doc['field_name'].value * factor", + "params" : { + "factor" : 1.1 + } }, "order" : "asc" } diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc index bb33628ba3b..b92dbfaa795 100644 --- a/docs/reference/search/search-template.asciidoc +++ b/docs/reference/search/search-template.asciidoc @@ -8,7 +8,7 @@ before they are executed and fill existing templates with template parameters. ------------------------------------------ GET /_search/template { - "template" : { + "inline" : { "query": { "match" : { "{{my_field}}" : "{{my_value}}" } }, "size" : "{{my_size}}" }, @@ -40,7 +40,7 @@ disable scripts per language, source and operation as described in ------------------------------------------ GET /_search/template { - "template": { + "inline": { "query": { "match": { "title": "{{query_string}}" @@ -60,7 +60,7 @@ GET /_search/template ------------------------------------------ GET /_search/template { - "template": { + "inline": { "query": { "terms": { "status": [ @@ -97,7 +97,7 @@ A default value is written as `{{var}}{{^var}}default{{/var}}` for instance: [source,js] ------------------------------------------ { - "template": { + "inline": { "query": { "range": { "line_no": { @@ -212,7 +212,7 @@ via the REST API, should be written as a string: [source,json] -------------------- -"template": "{\"query\":{\"filtered\":{\"query\":{\"match\":{\"line\":\"{{text}}\"}},\"filter\":{{{#line_no}}\"range\":{\"line_no\":{{{#start}}\"gte\":\"{{start}}\"{{#end}},{{/end}}{{/start}}{{#end}}\"lte\":\"{{end}}\"{{/end}}}}{{/line_no}}}}}}" +"inline": "{\"query\":{\"filtered\":{\"query\":{\"match\":{\"line\":\"{{text}}\"}},\"filter\":{{{#line_no}}\"range\":{\"line_no\":{{{#start}}\"gte\":\"{{start}}\"{{#end}},{{/end}}{{/start}}{{#end}}\"lte\":\"{{end}}\"{{/end}}}}{{/line_no}}}}}}" -------------------- ================================== @@ -229,9 +229,7 @@ In order to execute the stored template, reference it by it's name under the `te ------------------------------------------ GET /_search/template { - "template": { - "file": "storedTemplate" <1> - }, + "file": "storedTemplate", <1> "params": { "query_string": "search for these words" } @@ -293,9 +291,7 @@ To use an indexed template at search time use: ------------------------------------------ GET /_search/template { - "template": { - "id": "templateName" <1> - }, + "id": "templateName", <1> "params": { "query_string": "search for these words" } diff --git a/docs/reference/search/suggesters/phrase-suggest.asciidoc b/docs/reference/search/suggesters/phrase-suggest.asciidoc index 07332568685..9710b6a3f48 100644 --- a/docs/reference/search/suggesters/phrase-suggest.asciidoc +++ b/docs/reference/search/suggesters/phrase-suggest.asciidoc @@ -162,13 +162,13 @@ can contain misspellings (See parameter descriptions below). is wrapped rather than each token. `collate`:: - Checks each suggestion against the specified `query` or `filter` to - prune suggestions for which no matching docs exist in the index. - The collate query for a suggestion is run only on the local shard from which - the suggestion has been generated from. Either a `query` or a `filter` must - be specified, and it is run as a <>. + Checks each suggestion against the specified `query` to prune suggestions + for which no matching docs exist in the index. The collate query for a + suggestion is run only on the local shard from which the suggestion has + been generated from. The `query` must be specified, and it is run as + a <>. The current suggestion is automatically made available as the `{{suggestion}}` - variable, which should be used in your query/filter. You can still specify + variable, which should be used in your query. You can still specify your own template `params` -- the `suggestion` value will be added to the variables you specify. Additionally, you can specify a `prune` to control if all phrase suggestions will be returned, when set to `true` the suggestions diff --git a/docs/reference/setup/configuration.asciidoc b/docs/reference/setup/configuration.asciidoc index b822b223ce5..2bff0957690 100644 --- a/docs/reference/setup/configuration.asciidoc +++ b/docs/reference/setup/configuration.asciidoc @@ -264,6 +264,29 @@ file which will resolve to an environment setting, for example: } -------------------------------------------------- +Additionally, for settings that you do not wish to store in the configuration +file, you can use the value `${prompt::text}` or `${prompt::secret}` and start +Elasticsearch in the foreground. `${prompt::secret}` has echoing disabled so +that the value entered will not be shown in your terminal; `${prompt::text}` +will allow you to see the value as you type it in. For example: + +[source,yaml] +-------------------------------------------------- +node: + name: ${prompt::text} +-------------------------------------------------- + +On execution of the `elasticsearch` command, you will be prompted to enter +the actual value like so: + +[source,sh] +-------------------------------------------------- +Enter value for [node.name]: +-------------------------------------------------- + +NOTE: Elasticsearch will not start if `${prompt::text}` or `${prompt::secret}` +is used in the settings and the process is run as a service or in the background. + The location of the configuration file can be set externally using a system property: diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 873e6e9e8ca..3a174484ef9 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -377,9 +377,9 @@ public class CreateIndexRequest extends AcknowledgedRequest * Sets the settings and mappings as a single source. */ @SuppressWarnings("unchecked") - public CreateIndexRequest source(Map source) { + public CreateIndexRequest source(Map source) { boolean found = false; - for (Map.Entry entry : source.entrySet()) { + for (Map.Entry entry : source.entrySet()) { String name = entry.getKey(); if (name.equals("settings")) { found = true; diff --git a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java index 12648db563a..637c6d7ba08 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -93,7 +93,7 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder source) { + public CreateIndexRequestBuilder setSettings(Map source) { request.settings(source); return this; } @@ -223,7 +223,7 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder source) { + public CreateIndexRequestBuilder setSource(Map source) { request.source(source); return this; } diff --git a/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 715c1d716d9..a562dc046b2 100644 --- a/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -20,7 +20,12 @@ package org.elasticsearch.action.bulk; import com.google.common.collect.Lists; -import org.elasticsearch.action.*; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.CompositeIndicesRequest; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; @@ -140,7 +145,7 @@ public class BulkRequest extends ActionRequest implements Composite sizeInBytes += request.upsertRequest().source().length(); } if (request.script() != null) { - sizeInBytes += request.script().length() * 2; + sizeInBytes += request.script().getScript().length() * 2; } return this; } diff --git a/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index cf494358628..5b6674e38a1 100644 --- a/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -90,7 +90,7 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder source) { + public IndexRequestBuilder setSource(Map source) { request.source(source); return this; } @@ -100,7 +100,7 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder source, XContentType contentType) { + public IndexRequestBuilder setSource(Map source, XContentType contentType) { request.source(source, contentType); return this; } diff --git a/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 8e1da31affa..90ceee99f90 100644 --- a/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -35,11 +35,13 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptService.ScriptType; +import org.elasticsearch.script.Template; +import org.elasticsearch.script.mustache.MustacheScriptEngineService; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; -import java.util.Collections; import java.util.Map; import static org.elasticsearch.search.Scroll.readScroll; @@ -69,9 +71,7 @@ public class SearchRequest extends ActionRequest implements Indic private String preference; private BytesReference templateSource; - private String templateName; - private ScriptService.ScriptType templateType; - private Map templateParams = Collections.emptyMap(); + private Template template; private BytesReference source; @@ -100,9 +100,7 @@ public class SearchRequest extends ActionRequest implements Indic this.routing = searchRequest.routing; this.preference = searchRequest.preference; this.templateSource = searchRequest.templateSource; - this.templateName = searchRequest.templateName; - this.templateType = searchRequest.templateType; - this.templateParams = searchRequest.templateParams; + this.template = searchRequest.template; this.source = searchRequest.source; this.extraSource = searchRequest.extraSource; this.queryCache = searchRequest.queryCache; @@ -390,42 +388,92 @@ public class SearchRequest extends ActionRequest implements Indic } /** - * The name of the stored template + * The stored template */ + public void template(Template template) { + this.template = template; + } + + /** + * The stored template + */ + public Template template() { + return template; + } + + /** + * The name of the stored template + * + * @deprecated use {@link #template(Template))} instead. + */ + @Deprecated public void templateName(String templateName) { - this.templateName = templateName; + updateOrCreateScript(templateName, null, null, null); } + /** + * The type of the stored template + * + * @deprecated use {@link #template(Template))} instead. + */ + @Deprecated public void templateType(ScriptService.ScriptType templateType) { - this.templateType = templateType; + updateOrCreateScript(null, templateType, null, null); } /** * Template parameters used for rendering + * + * @deprecated use {@link #template(Template))} instead. */ + @Deprecated public void templateParams(Map params) { - this.templateParams = params; + updateOrCreateScript(null, null, null, params); } /** * The name of the stored template + * + * @deprecated use {@link #template()} instead. */ + @Deprecated public String templateName() { - return templateName; + return template == null ? null : template.getScript(); } /** * The name of the stored template + * + * @deprecated use {@link #template()} instead. */ + @Deprecated public ScriptService.ScriptType templateType() { - return templateType; + return template == null ? null : template.getType(); } /** * Template parameters used for rendering + * + * @deprecated use {@link #template()} instead. */ + @Deprecated public Map templateParams() { - return templateParams; + return template == null ? null : template.getParams(); + } + + private void updateOrCreateScript(String templateContent, ScriptType type, String lang, Map params) { + Template template = template(); + if (template == null) { + template = new Template(templateContent == null ? "" : templateContent, type == null ? ScriptType.INLINE : type, lang, null, + params); + } else { + String newTemplateContent = templateContent == null ? template.getScript() : templateContent; + ScriptType newTemplateType = type == null ? template.getType() : type; + String newTemplateLang = lang == null ? template.getLang() : lang; + Map newTemplateParams = params == null ? template.getParams() : params; + template = new Template(newTemplateContent, newTemplateType, MustacheScriptEngineService.NAME, null, newTemplateParams); + } + template(template); } /** @@ -517,10 +565,8 @@ public class SearchRequest extends ActionRequest implements Indic indicesOptions = IndicesOptions.readIndicesOptions(in); templateSource = in.readBytesReference(); - templateName = in.readOptionalString(); - templateType = ScriptService.ScriptType.readFrom(in); if (in.readBoolean()) { - templateParams = (Map) in.readGenericValue(); + template = Template.readTemplate(in); } queryCache = in.readOptionalBoolean(); } @@ -550,12 +596,10 @@ public class SearchRequest extends ActionRequest implements Indic indicesOptions.writeIndicesOptions(out); out.writeBytesReference(templateSource); - out.writeOptionalString(templateName); - ScriptService.ScriptType.writeTo(templateType, out); - boolean existTemplateParams = templateParams != null; - out.writeBoolean(existTemplateParams); - if (existTemplateParams) { - out.writeGenericValue(templateParams); + boolean hasTemplate = template != null; + out.writeBoolean(hasTemplate); + if (hasTemplate) { + template.writeTo(out); } out.writeOptionalBoolean(queryCache); diff --git a/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index f7e84b0733a..690ad43b466 100644 --- a/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -29,7 +29,9 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.Template; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -423,33 +425,60 @@ public class SearchRequestBuilder extends ActionRequestBuilder params) { sourceBuilder().scriptField(name, script, params); return this; } /** - * Adds a script based field to load and return. The field does not have to be stored, - * but its recommended to use non analyzed or numeric fields. + * Adds a script based field to load and return. The field does not have to + * be stored, but its recommended to use non analyzed or numeric fields. * - * @param name The name that will represent this value in the return hit - * @param lang The language of the script - * @param script The script to use - * @param params Parameters that the script can use (can be null). + * @param name + * The name that will represent this value in the return hit + * @param lang + * The language of the script + * @param script + * The script to use + * @param params + * Parameters that the script can use (can be null). + * @deprecated Use {@link #addScriptField(String, Script)} instead. */ + @Deprecated public SearchRequestBuilder addScriptField(String name, String lang, String script, Map params) { sourceBuilder().scriptField(name, lang, script, params); return this; @@ -939,16 +968,33 @@ public class SearchRequestBuilder extends ActionRequestBuilder templateParams) { request.templateParams(templateParams); return this; diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryAndFetchAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryAndFetchAction.java index 83e9aba54f0..01eefdb564d 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryAndFetchAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryAndFetchAction.java @@ -91,7 +91,7 @@ public class TransportSearchDfsQueryAndFetchAction extends TransportSearchTypeAc } } - void executeSecondPhase(final int shardIndex, final DfsSearchResult dfsResult, final AtomicInteger counter, DiscoveryNode node, final QuerySearchRequest querySearchRequest) { + void executeSecondPhase(final int shardIndex, final DfsSearchResult dfsResult, final AtomicInteger counter, final DiscoveryNode node, final QuerySearchRequest querySearchRequest) { searchService.sendExecuteFetch(node, querySearchRequest, new ActionListener() { @Override public void onResponse(QueryFetchSearchResult result) { @@ -104,7 +104,14 @@ public class TransportSearchDfsQueryAndFetchAction extends TransportSearchTypeAc @Override public void onFailure(Throwable t) { - onSecondPhaseFailure(t, querySearchRequest, shardIndex, dfsResult, counter); + try { + onSecondPhaseFailure(t, querySearchRequest, shardIndex, dfsResult, counter); + } finally { + // the query might not have been executed at all (for example because thread pool rejected execution) + // and the search context that was created in dfs phase might not be released. + // release it again to be in the safe side + sendReleaseSearchContext(querySearchRequest.id(), node); + } } }); } diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java index c1a361903e8..3a4e6d70533 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchDfsQueryThenFetchAction.java @@ -100,7 +100,7 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA } } - void executeQuery(final int shardIndex, final DfsSearchResult dfsResult, final AtomicInteger counter, final QuerySearchRequest querySearchRequest, DiscoveryNode node) { + void executeQuery(final int shardIndex, final DfsSearchResult dfsResult, final AtomicInteger counter, final QuerySearchRequest querySearchRequest, final DiscoveryNode node) { searchService.sendExecuteQuery(node, querySearchRequest, new ActionListener() { @Override public void onResponse(QuerySearchResult result) { @@ -113,7 +113,14 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA @Override public void onFailure(Throwable t) { - onQueryFailure(t, querySearchRequest, shardIndex, dfsResult, counter); + try { + onQueryFailure(t, querySearchRequest, shardIndex, dfsResult, counter); + } finally { + // the query might not have been executed at all (for example because thread pool rejected execution) + // and the search context that was created in dfs phase might not be released. + // release it again to be in the safe side + sendReleaseSearchContext(querySearchRequest.id(), node); + } } }); } @@ -176,6 +183,11 @@ public class TransportSearchDfsQueryThenFetchAction extends TransportSearchTypeA @Override public void onFailure(Throwable t) { + // the search context might not be cleared on the node where the fetch was executed for example + // because the action was rejected by the thread pool. in this case we need to send a dedicated + // request to clear the search context. by setting docIdsToLoad to null, the context will be cleared + // in TransportSearchTypeAction.releaseIrrelevantSearchContexts() after the search request is done. + docIdsToLoad.set(shardIndex, null); onFetchFailure(t, fetchSearchRequest, shardIndex, shardTarget, counter); } }); diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java index 175a770e9c6..79a164c12ef 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchQueryThenFetchAction.java @@ -35,8 +35,8 @@ import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.action.SearchServiceTransportAction; import org.elasticsearch.search.controller.SearchPhaseController; -import org.elasticsearch.search.fetch.ShardFetchSearchRequest; import org.elasticsearch.search.fetch.FetchSearchResult; +import org.elasticsearch.search.fetch.ShardFetchSearchRequest; import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.search.internal.ShardSearchTransportRequest; import org.elasticsearch.search.query.QuerySearchResultProvider; @@ -118,7 +118,10 @@ public class TransportSearchQueryThenFetchAction extends TransportSearchTypeActi @Override public void onFailure(Throwable t) { - // the failure might happen without managing to clear the search context..., potentially need to clear its context (for example) + // the search context might not be cleared on the node where the fetch was executed for example + // because the action was rejected by the thread pool. in this case we need to send a dedicated + // request to clear the search context. by setting docIdsToLoad to null, the context will be cleared + // in TransportSearchTypeAction.releaseIrrelevantSearchContexts() after the search request is done. docIdsToLoad.set(shardIndex, null); onFetchFailure(t, fetchSearchRequest, shardIndex, shardTarget, counter); } diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java index e2cf4d87f53..4c1210920d7 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchTypeAction.java @@ -303,9 +303,7 @@ public abstract class TransportSearchTypeAction extends TransportAction entry : firstResults.asList()) { try { DiscoveryNode node = nodes.get(entry.value.shardTarget().nodeId()); - if (node != null) { // should not happen (==null) but safeguard anyhow - searchService.sendFreeContext(node, entry.value.id(), request); - } + sendReleaseSearchContext(entry.value.id(), node); } catch (Throwable t1) { logger.trace("failed to release context", t1); } @@ -329,9 +327,7 @@ public abstract class TransportSearchTypeAction extends TransportAction entry, ScoreDoc[] lastEmittedDocPerShard) { if (lastEmittedDocPerShard != null) { ScoreDoc lastEmittedDoc = lastEmittedDocPerShard[entry.index]; diff --git a/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 2e2a9e7abf3..d3f7a5b9356 100644 --- a/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -43,7 +43,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.settings.Settings; @@ -1080,7 +1080,7 @@ public abstract class TransportReplicationAction private String parent; @Nullable - String script; - @Nullable - ScriptService.ScriptType scriptType; - @Nullable - String scriptLang; - @Nullable - Map scriptParams; + Script script; private String[] fields; @@ -205,105 +200,171 @@ public class UpdateRequest extends InstanceShardOperationRequest return this.shardId; } - public String script() { + public Script script() { return this.script; } - public ScriptService.ScriptType scriptType() { return this.scriptType; } + /** + * The script to execute. Note, make sure not to send different script each times and instead + * use script params if possible with the same (automatically compiled) script. + */ + public UpdateRequest script(Script script) { + this.script = script; + return this; + } + /** + * @deprecated Use {@link #script()} instead + */ + @Deprecated + public String scriptString() { + return this.script == null ? null : this.script.getScript(); + } + + /** + * @deprecated Use {@link #script()} instead + */ + @Deprecated + public ScriptService.ScriptType scriptType() { + return this.script == null ? null : this.script.getType(); + } + + /** + * @deprecated Use {@link #script()} instead + */ + @Deprecated public Map scriptParams() { - return this.scriptParams; + return this.script == null ? null : this.script.getParams(); } /** - * The script to execute. Note, make sure not to send different script each times and instead - * use script params if possible with the same (automatically compiled) script. + * The script to execute. Note, make sure not to send different script each + * times and instead use script params if possible with the same + * (automatically compiled) script. + * + * @deprecated Use {@link #script(Script)} instead */ + @Deprecated public UpdateRequest script(String script, ScriptService.ScriptType scriptType) { - this.script = script; - this.scriptType = scriptType; + updateOrCreateScript(script, scriptType, null, null); return this; } /** - * The script to execute. Note, make sure not to send different script each times and instead - * use script params if possible with the same (automatically compiled) script. + * The script to execute. Note, make sure not to send different script each + * times and instead use script params if possible with the same + * (automatically compiled) script. + * + * @deprecated Use {@link #script(Script)} instead */ + @Deprecated public UpdateRequest script(String script) { - this.script = script; - this.scriptType = ScriptService.ScriptType.INLINE; + updateOrCreateScript(script, ScriptType.INLINE, null, null); return this; } - /** * The language of the script to execute. + * + * @deprecated Use {@link #script(Script)} instead */ + @Deprecated public UpdateRequest scriptLang(String scriptLang) { - this.scriptLang = scriptLang; + updateOrCreateScript(null, null, scriptLang, null); return this; } + /** + * @deprecated Use {@link #script()} instead + */ + @Deprecated public String scriptLang() { - return scriptLang; + return script == null ? null : script.getLang(); } /** * Add a script parameter. + * + * @deprecated Use {@link #script(Script)} instead */ + @Deprecated public UpdateRequest addScriptParam(String name, Object value) { - if (scriptParams == null) { - scriptParams = Maps.newHashMap(); + Script script = script(); + if (script == null) { + HashMap scriptParams = new HashMap(); + scriptParams.put(name, value); + updateOrCreateScript(null, null, null, scriptParams); + } else { + Map scriptParams = script.getParams(); + if (scriptParams == null) { + scriptParams = new HashMap(); + scriptParams.put(name, value); + updateOrCreateScript(null, null, null, scriptParams); + } else { + scriptParams.put(name, value); + } } - scriptParams.put(name, value); return this; } /** * Sets the script parameters to use with the script. + * + * @deprecated Use {@link #script(Script)} instead */ + @Deprecated public UpdateRequest scriptParams(Map scriptParams) { - if (this.scriptParams == null) { - this.scriptParams = scriptParams; - } else { - this.scriptParams.putAll(scriptParams); - } + updateOrCreateScript(null, null, null, scriptParams); return this; } + private void updateOrCreateScript(String scriptContent, ScriptType type, String lang, Map params) { + Script script = script(); + if (script == null) { + script = new Script(scriptContent == null ? "" : scriptContent, type == null ? ScriptType.INLINE : type, lang, params); + } else { + String newScriptContent = scriptContent == null ? script.getScript() : scriptContent; + ScriptType newScriptType = type == null ? script.getType() : type; + String newScriptLang = lang == null ? script.getLang() : lang; + Map newScriptParams = params == null ? script.getParams() : params; + script = new Script(newScriptContent, newScriptType, newScriptLang, newScriptParams); + } + script(script); + } + /** - * The script to execute. Note, make sure not to send different script each times and instead - * use script params if possible with the same (automatically compiled) script. + * The script to execute. Note, make sure not to send different script each + * times and instead use script params if possible with the same + * (automatically compiled) script. + * + * @deprecated Use {@link #script(Script)} instead */ + @Deprecated public UpdateRequest script(String script, ScriptService.ScriptType scriptType, @Nullable Map scriptParams) { - this.script = script; - this.scriptType = scriptType; - if (this.scriptParams != null) { - this.scriptParams.putAll(scriptParams); - } else { - this.scriptParams = scriptParams; - } + this.script = new Script(script, scriptType, null, scriptParams); return this; } /** - * The script to execute. Note, make sure not to send different script each times and instead - * use script params if possible with the same (automatically compiled) script. + * The script to execute. Note, make sure not to send different script each + * times and instead use script params if possible with the same + * (automatically compiled) script. * - * @param script The script to execute - * @param scriptLang The script language - * @param scriptType The script type - * @param scriptParams The script parameters + * @param script + * The script to execute + * @param scriptLang + * The script language + * @param scriptType + * The script type + * @param scriptParams + * The script parameters + * + * @deprecated Use {@link #script(Script)} instead */ - public UpdateRequest script(String script, @Nullable String scriptLang, ScriptService.ScriptType scriptType, @Nullable Map scriptParams) { - this.script = script; - this.scriptLang = scriptLang; - this.scriptType = scriptType; - if (this.scriptParams != null) { - this.scriptParams.putAll(scriptParams); - } else { - this.scriptParams = scriptParams; - } + @Deprecated + public UpdateRequest script(String script, @Nullable String scriptLang, ScriptService.ScriptType scriptType, + @Nullable Map scriptParams) { + this.script = new Script(script, scriptType, scriptLang, scriptParams); return this; } @@ -574,6 +635,8 @@ public class UpdateRequest extends InstanceShardOperationRequest public UpdateRequest source(BytesReference source) throws Exception { ScriptParameterParser scriptParameterParser = new ScriptParameterParser(); + Map scriptParams = null; + Script script = null; XContentType xContentType = XContentFactory.xContentType(source); try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(source)) { XContentParser.Token token = parser.nextToken(); @@ -584,6 +647,8 @@ public class UpdateRequest extends InstanceShardOperationRequest while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); + } else if ("script".equals(currentFieldName) && token == XContentParser.Token.START_OBJECT) { + script = Script.parse(parser); } else if ("params".equals(currentFieldName)) { scriptParams = parser.map(); } else if ("scripted_upsert".equals(currentFieldName)) { @@ -604,12 +669,16 @@ public class UpdateRequest extends InstanceShardOperationRequest scriptParameterParser.token(currentFieldName, token, parser); } } - ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue(); - if (scriptValue != null) { - script = scriptValue.script(); - scriptType = scriptValue.scriptType(); + // Don't have a script using the new API so see if it is specified with the old API + if (script == null) { + ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue(); + if (scriptValue != null) { + script = new Script(scriptValue.script(), scriptValue.scriptType(), scriptParameterParser.lang(), scriptParams); + } + } + if (script != null) { + this.script = script; } - scriptLang = scriptParameterParser.lang(); } return this; } @@ -639,12 +708,9 @@ public class UpdateRequest extends InstanceShardOperationRequest id = in.readString(); routing = in.readOptionalString(); parent = in.readOptionalString(); - script = in.readOptionalString(); - if(Strings.hasLength(script)) { - scriptType = ScriptService.ScriptType.readFrom(in); + if (in.readBoolean()) { + script = Script.readScript(in); } - scriptLang = in.readOptionalString(); - scriptParams = in.readMap(); retryOnConflict = in.readVInt(); refresh = in.readBoolean(); if (in.readBoolean()) { @@ -677,12 +743,11 @@ public class UpdateRequest extends InstanceShardOperationRequest out.writeString(id); out.writeOptionalString(routing); out.writeOptionalString(parent); - out.writeOptionalString(script); - if (Strings.hasLength(script)) { - ScriptService.ScriptType.writeTo(scriptType, out); + boolean hasScript = script != null; + out.writeBoolean(hasScript); + if (hasScript) { + script.writeTo(out); } - out.writeOptionalString(scriptLang); - out.writeMap(scriptParams); out.writeVInt(retryOnConflict); out.writeBoolean(refresh); if (doc == null) { diff --git a/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index 3bcb9c640df..7c30c47dd7c 100644 --- a/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import java.util.Map; @@ -80,21 +81,43 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilderctx, which is bound to the entry, * e.g. ctx._source.mycounter += 1. * + */ + public UpdateRequestBuilder setScript(Script script) { + request.script(script); + return this; + } + + /** + * The script to execute. Note, make sure not to send different script each + * times and instead use script params if possible with the same + * (automatically compiled) script. + *

+ * The script works with the variable ctx, which is bound to + * the entry, e.g. ctx._source.mycounter += 1. + * * @see #setScriptLang(String) * @see #setScriptParams(Map) + * + * @deprecated use {@link #setScript(Script)} instead */ + @Deprecated public UpdateRequestBuilder setScript(String script, ScriptService.ScriptType scriptType) { request.script(script, scriptType); return this; } /** - * The language of the script to execute. - * Valid options are: mvel, js, groovy, python, and native (Java)
+ * The language of the script to execute. Valid options are: mvel, js, + * groovy, python, and native (Java)
* Default: groovy *

- * Ref: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-scripting.html + * Ref: + * http://www.elasticsearch.org/guide/en/elasticsearch/reference/current + * /modules-scripting.html + * + * @deprecated use {@link #setScript(Script)} instead */ + @Deprecated public UpdateRequestBuilder setScriptLang(String scriptLang) { request.scriptLang(scriptLang); return this; @@ -102,7 +125,10 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder scriptParams) { request.scriptParams(scriptParams); return this; @@ -110,7 +136,10 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder initialSettings() { - return InternalSettingsPreparer.prepareSettings(EMPTY_SETTINGS, true); + private static Tuple initialSettings(boolean foreground) { + Terminal terminal = foreground ? Terminal.DEFAULT : null; + return InternalSettingsPreparer.prepareSettings(EMPTY_SETTINGS, true, terminal); } private void start() { @@ -226,7 +237,7 @@ public class Bootstrap { Settings settings = null; Environment environment = null; try { - Tuple tuple = initialSettings(); + Tuple tuple = initialSettings(foreground); settings = tuple.v1(); environment = tuple.v2(); diff --git a/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java b/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java index 97bf98e60f6..226fea665cd 100644 --- a/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java +++ b/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java @@ -20,33 +20,53 @@ package org.elasticsearch.bootstrap; import com.sun.jna.Native; +import com.sun.jna.Structure; + +import org.apache.lucene.util.Constants; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; +import java.util.Arrays; +import java.util.List; /** - * + * java mapping to some libc functions */ -class JNACLibrary { +final class JNACLibrary { private static final ESLogger logger = Loggers.getLogger(JNACLibrary.class); public static final int MCL_CURRENT = 1; - public static final int MCL_FUTURE = 2; - public static final int ENOMEM = 12; + public static final int RLIMIT_MEMLOCK = Constants.MAC_OS_X ? 6 : 8; + public static final long RLIM_INFINITY = Constants.MAC_OS_X ? 9223372036854775807L : -1L; static { try { Native.register("c"); } catch (UnsatisfiedLinkError e) { - logger.warn("unable to link C library. native methods (mlockall) will be disabled."); + logger.warn("unable to link C library. native methods (mlockall) will be disabled.", e); } } static native int mlockall(int flags); static native int geteuid(); + + /** corresponds to struct rlimit */ + public static final class Rlimit extends Structure implements Structure.ByReference { + public long rlim_cur = 0; + public long rlim_max = 0; + + @Override + protected List getFieldOrder() { + return Arrays.asList(new String[] { "rlim_cur", "rlim_max" }); + } + } + + static native int getrlimit(int resource, Rlimit rlimit); + + static native String strerror(int errno); private JNACLibrary() { } diff --git a/src/main/java/org/elasticsearch/bootstrap/JNANatives.java b/src/main/java/org/elasticsearch/bootstrap/JNANatives.java index eb29df85cdb..ba6eef16257 100644 --- a/src/main/java/org/elasticsearch/bootstrap/JNANatives.java +++ b/src/main/java/org/elasticsearch/bootstrap/JNANatives.java @@ -26,8 +26,6 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.monitor.jvm.JvmInfo; -import java.util.Locale; - import static org.elasticsearch.bootstrap.JNAKernel32Library.SizeT; /** @@ -43,30 +41,66 @@ class JNANatives { static void tryMlockall() { int errno = Integer.MIN_VALUE; + String errMsg = null; + boolean rlimitSuccess = false; + long softLimit = 0; + long hardLimit = 0; + try { int result = JNACLibrary.mlockall(JNACLibrary.MCL_CURRENT); - if (result != 0) { - errno = Native.getLastError(); - } else { + if (result == 0) { LOCAL_MLOCKALL = true; + return; + } + + errno = Native.getLastError(); + errMsg = JNACLibrary.strerror(errno); + if (Constants.LINUX || Constants.MAC_OS_X) { + // we only know RLIMIT_MEMLOCK for these two at the moment. + JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit(); + if (JNACLibrary.getrlimit(JNACLibrary.RLIMIT_MEMLOCK, rlimit) == 0) { + rlimitSuccess = true; + softLimit = rlimit.rlim_cur; + hardLimit = rlimit.rlim_max; + } else { + logger.warn("Unable to retrieve resource limits: " + JNACLibrary.strerror(Native.getLastError())); + } } } catch (UnsatisfiedLinkError e) { // this will have already been logged by CLibrary, no need to repeat it return; } - if (errno != Integer.MIN_VALUE) { - if (errno == JNACLibrary.ENOMEM && System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("linux")) { - logger.warn("Unable to lock JVM memory (ENOMEM)." - + " This can result in part of the JVM being swapped out." - + " Increase RLIMIT_MEMLOCK (ulimit)."); - } else if (!System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("mac")) { - // OS X allows mlockall to be called, but always returns an error - logger.warn("Unknown mlockall error " + errno); + // mlockall failed for some reason + logger.warn("Unable to lock JVM Memory: error=" + errno + ",reason=" + errMsg + ". This can result in part of the JVM being swapped out."); + if (errno == JNACLibrary.ENOMEM) { + if (rlimitSuccess) { + logger.warn("Increase RLIMIT_MEMLOCK, soft limit: " + rlimitToString(softLimit) + ", hard limit: " + rlimitToString(hardLimit)); + if (Constants.LINUX) { + // give specific instructions for the linux case to make it easy + logger.warn("These can be adjusted by modifying /etc/security/limits.conf, for example: \n" + + "\t# allow user 'esuser' mlockall\n" + + "\tesuser soft memlock unlimited\n" + + "\tesuser hard memlock unlimited" + ); + logger.warn("If you are logged in interactively, you will have to re-login for the new limits to take effect."); + } + } else { + logger.warn("Increase RLIMIT_MEMLOCK (ulimit)."); } } } + static String rlimitToString(long value) { + assert Constants.LINUX || Constants.MAC_OS_X; + if (value == JNACLibrary.RLIM_INFINITY) { + return "unlimited"; + } else { + // TODO, on java 8 use Long.toUnsignedString, since thats what it is. + return Long.toString(value); + } + } + /** Returns true if user is root, false if not, or if we don't know */ static boolean definitelyRunningAsRoot() { if (Constants.WINDOWS) { diff --git a/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/src/main/java/org/elasticsearch/client/transport/TransportClient.java index 9e3cb1f0f80..d63e94d2ffe 100644 --- a/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -20,8 +20,14 @@ package org.elasticsearch.client.transport; import com.google.common.collect.ImmutableList; + import org.elasticsearch.Version; -import org.elasticsearch.action.*; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionModule; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cache.recycler.PageCacheRecycler; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.client.support.Headers; @@ -30,7 +36,6 @@ import org.elasticsearch.cluster.ClusterNameModule; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.LifecycleComponent; -import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.network.NetworkModule; @@ -122,8 +127,6 @@ public class TransportClient extends AbstractClient { Version version = Version.CURRENT; - CompressorFactory.configure(this.settings); - final ThreadPool threadPool = new ThreadPool(settings); boolean success = false; diff --git a/src/main/java/org/elasticsearch/cluster/ClusterState.java b/src/main/java/org/elasticsearch/cluster/ClusterState.java index 85a2313daeb..cb16fd8dc7b 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -37,7 +37,7 @@ import org.elasticsearch.cluster.service.InternalClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -422,7 +422,7 @@ public class ClusterState implements ToXContent, Diffable { builder.endObject(); builder.startObject("mappings"); - for (ObjectObjectCursor cursor1 : templateMetaData.mappings()) { + for (ObjectObjectCursor cursor1 : templateMetaData.mappings()) { byte[] mappingSource = cursor1.value.uncompressed(); XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); Map mapping = parser.map(); diff --git a/src/main/java/org/elasticsearch/cluster/DiskUsage.java b/src/main/java/org/elasticsearch/cluster/DiskUsage.java index 29cac6d7b7c..92725b08831 100644 --- a/src/main/java/org/elasticsearch/cluster/DiskUsage.java +++ b/src/main/java/org/elasticsearch/cluster/DiskUsage.java @@ -59,6 +59,10 @@ public class DiskUsage { return 100.0 * ((double)freeBytes / totalBytes); } + public double getUsedDiskAsPercentage() { + return 100.0 - getFreeDiskAsPercentage(); + } + public long getFreeBytes() { return freeBytes; } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index 0f7e55c8087..fb640eedc5a 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableSet; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; @@ -45,7 +45,7 @@ public class AliasMetaData extends AbstractDiffable { private final String alias; - private final CompressedString filter; + private final CompressedXContent filter; private final String indexRouting; @@ -53,7 +53,7 @@ public class AliasMetaData extends AbstractDiffable { private final Set searchRoutingValues; - private AliasMetaData(String alias, CompressedString filter, String indexRouting, String searchRouting) { + private AliasMetaData(String alias, CompressedXContent filter, String indexRouting, String searchRouting) { this.alias = alias; this.filter = filter; this.indexRouting = indexRouting; @@ -77,11 +77,11 @@ public class AliasMetaData extends AbstractDiffable { return alias(); } - public CompressedString filter() { + public CompressedXContent filter() { return filter; } - public CompressedString getFilter() { + public CompressedXContent getFilter() { return filter(); } @@ -176,9 +176,9 @@ public class AliasMetaData extends AbstractDiffable { @Override public AliasMetaData readFrom(StreamInput in) throws IOException { String alias = in.readString(); - CompressedString filter = null; + CompressedXContent filter = null; if (in.readBoolean()) { - filter = CompressedString.readCompressedString(in); + filter = CompressedXContent.readCompressedString(in); } String indexRouting = null; if (in.readBoolean()) { @@ -195,7 +195,7 @@ public class AliasMetaData extends AbstractDiffable { private final String alias; - private CompressedString filter; + private CompressedXContent filter; private String indexRouting; @@ -217,7 +217,7 @@ public class AliasMetaData extends AbstractDiffable { return alias; } - public Builder filter(CompressedString filter) { + public Builder filter(CompressedXContent filter) { this.filter = filter; return this; } @@ -244,7 +244,7 @@ public class AliasMetaData extends AbstractDiffable { } try { XContentBuilder builder = XContentFactory.jsonBuilder().map(filter); - this.filter = new CompressedString(builder.bytes()); + this.filter = new CompressedXContent(builder.bytes()); return this; } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to build json for alias request", e); @@ -324,7 +324,7 @@ public class AliasMetaData extends AbstractDiffable { } } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { if ("filter".equals(currentFieldName)) { - builder.filter(new CompressedString(parser.binaryValue())); + builder.filter(new CompressedXContent(parser.binaryValue())); } } else if (token == XContentParser.Token.VALUE_STRING) { if ("routing".equals(currentFieldName)) { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 07703bca591..2f3f6c889f8 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -35,7 +35,7 @@ import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -874,7 +874,7 @@ public class IndexMetaData implements Diffable { if ("mappings".equals(currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { - builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue()))); + builder.putMapping(new MappingMetaData(new CompressedXContent(parser.binaryValue()))); } else { Map mapping = parser.mapOrdered(); if (mapping.size() == 1) { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index ae555a54e75..d91d0817cfc 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -24,7 +24,7 @@ import com.google.common.collect.Sets; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -54,13 +54,13 @@ public class IndexTemplateMetaData extends AbstractDiffable mappings; + private final ImmutableOpenMap mappings; private final ImmutableOpenMap aliases; private final ImmutableOpenMap customs; - public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableOpenMap mappings, + public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableOpenMap mappings, ImmutableOpenMap aliases, ImmutableOpenMap customs) { this.name = name; this.order = order; @@ -103,11 +103,11 @@ public class IndexTemplateMetaData extends AbstractDiffable mappings() { + public ImmutableOpenMap mappings() { return this.mappings; } - public ImmutableOpenMap getMappings() { + public ImmutableOpenMap getMappings() { return this.mappings; } @@ -170,7 +170,7 @@ public class IndexTemplateMetaData extends AbstractDiffable cursor : mappings) { + for (ObjectObjectCursor cursor : mappings) { out.writeString(cursor.key); cursor.value.writeTo(out); } @@ -223,7 +223,7 @@ public class IndexTemplateMetaData extends AbstractDiffable mappings; + private final ImmutableOpenMap.Builder mappings; private final ImmutableOpenMap.Builder aliases; @@ -276,13 +276,13 @@ public class IndexTemplateMetaData extends AbstractDiffable cursor : indexTemplateMetaData.mappings()) { + for (ObjectObjectCursor cursor : indexTemplateMetaData.mappings()) { byte[] mappingSource = cursor.value.uncompressed(); XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource); Map mapping = parser.map(); @@ -341,7 +341,7 @@ public class IndexTemplateMetaData extends AbstractDiffable cursor : indexTemplateMetaData.mappings()) { + for (ObjectObjectCursor cursor : indexTemplateMetaData.mappings()) { byte[] data = cursor.value.uncompressed(); XContentParser parser = XContentFactory.xContent(data).createParser(data); Map mapping = parser.mapOrderedAndClose(); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java index e6067c46817..2d8054d748f 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java @@ -23,7 +23,7 @@ import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.joda.FormatDateTimeFormatter; @@ -276,7 +276,7 @@ public class MappingMetaData extends AbstractDiffable { private final String type; - private final CompressedString source; + private final CompressedXContent source; private Id id; private Routing routing; @@ -294,9 +294,9 @@ public class MappingMetaData extends AbstractDiffable { this.hasParentField = docMapper.parentFieldMapper().active(); } - public MappingMetaData(CompressedString mapping) throws IOException { + public MappingMetaData(CompressedXContent mapping) throws IOException { this.source = mapping; - Map mappingMap = XContentHelper.createParser(mapping.compressed(), 0, mapping.compressed().length).mapOrderedAndClose(); + Map mappingMap = XContentHelper.createParser(mapping.compressedReference()).mapOrderedAndClose(); if (mappingMap.size() != 1) { throw new IllegalStateException("Can't derive type from mapping, no root type: " + mapping.string()); } @@ -311,7 +311,7 @@ public class MappingMetaData extends AbstractDiffable { public MappingMetaData(String type, Map mapping) throws IOException { this.type = type; XContentBuilder mappingBuilder = XContentFactory.jsonBuilder().map(mapping); - this.source = new CompressedString(mappingBuilder.bytes()); + this.source = new CompressedXContent(mappingBuilder.bytes()); Map withoutType = mapping; if (mapping.size() == 1 && mapping.containsKey(type)) { withoutType = (Map) mapping.get(type); @@ -322,7 +322,7 @@ public class MappingMetaData extends AbstractDiffable { private MappingMetaData() { this.type = ""; try { - this.source = new CompressedString(""); + this.source = new CompressedXContent("{}"); } catch (IOException ex) { throw new IllegalStateException("Cannot create MappingMetaData prototype", ex); } @@ -393,7 +393,7 @@ public class MappingMetaData extends AbstractDiffable { } } - public MappingMetaData(String type, CompressedString source, Id id, Routing routing, Timestamp timestamp, boolean hasParentField) { + public MappingMetaData(String type, CompressedXContent source, Id id, Routing routing, Timestamp timestamp, boolean hasParentField) { this.type = type; this.source = source; this.id = id; @@ -418,7 +418,7 @@ public class MappingMetaData extends AbstractDiffable { return this.type; } - public CompressedString source() { + public CompressedXContent source() { return this.source; } @@ -430,7 +430,7 @@ public class MappingMetaData extends AbstractDiffable { * Converts the serialized compressed form of the mappings into a parsed map. */ public Map sourceAsMap() throws IOException { - Map mapping = XContentHelper.convertToMap(source.compressed(), 0, source.compressed().length, true).v2(); + Map mapping = XContentHelper.convertToMap(source.compressedReference(), true).v2(); if (mapping.size() == 1 && mapping.containsKey(type())) { // the type name is the root value, reduce it mapping = (Map) mapping.get(type()); @@ -599,7 +599,7 @@ public class MappingMetaData extends AbstractDiffable { public MappingMetaData readFrom(StreamInput in) throws IOException { String type = in.readString(); - CompressedString source = CompressedString.readCompressedString(in); + CompressedXContent source = CompressedXContent.readCompressedString(in); // id Id id = new Id(in.readBoolean() ? in.readString() : null); // routing diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 41e310a95ad..b4ca88f046b 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -46,7 +46,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.regex.Regex; @@ -87,7 +87,6 @@ public class MetaDataCreateIndexService extends AbstractComponent { public final static int MAX_INDEX_NAME_BYTES = 255; private static final DefaultIndexTemplateFilter DEFAULT_INDEX_TEMPLATE_FILTER = new DefaultIndexTemplateFilter(); - private final Environment environment; private final ThreadPool threadPool; private final ClusterService clusterService; private final IndicesService indicesService; @@ -100,12 +99,11 @@ public class MetaDataCreateIndexService extends AbstractComponent { private final NodeEnvironment nodeEnv; @Inject - public MetaDataCreateIndexService(Settings settings, Environment environment, ThreadPool threadPool, ClusterService clusterService, + public MetaDataCreateIndexService(Settings settings, ThreadPool threadPool, ClusterService clusterService, IndicesService indicesService, AllocationService allocationService, MetaDataService metaDataService, Version version, @RiverIndexName String riverIndexName, AliasValidator aliasValidator, Set indexTemplateFilters, NodeEnvironment nodeEnv) { super(settings); - this.environment = environment; this.threadPool = threadPool; this.clusterService = clusterService; this.indicesService = indicesService; @@ -254,7 +252,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { // apply templates, merging the mappings into the request mapping if exists for (IndexTemplateMetaData template : templates) { templateNames.add(template.getName()); - for (ObjectObjectCursor cursor : template.mappings()) { + for (ObjectObjectCursor cursor : template.mappings()) { if (mappings.containsKey(cursor.key)) { XContentHelper.mergeDefaults(mappings.get(cursor.key), parseMapping(cursor.value.string())); } else { @@ -357,7 +355,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { // first, add the default mapping if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) { try { - mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedString(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), false); + mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), false); } catch (Exception e) { removalReason = "failed on parsing default mapping on index creation"; throw new MapperParsingException("mapping [" + MapperService.DEFAULT_MAPPING + "]", e); @@ -369,7 +367,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { } try { // apply the default here, its the first time we parse it - mapperService.merge(entry.getKey(), new CompressedString(XContentFactory.jsonBuilder().map(entry.getValue()).string()), true); + mapperService.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), true); } catch (Exception e) { removalReason = "failed on parsing mappings on index creation"; throw new MapperParsingException("mapping [" + entry.getKey() + "]", e); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index 0e39e7a613d..ae6f32edd13 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -19,9 +19,11 @@ package org.elasticsearch.cluster.metadata; +import com.carrotsearch.hppc.cursors.ObjectCursor; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingClusterStateUpdateRequest; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; @@ -32,14 +34,14 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.common.Priority; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MergeMappingException; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MergeResult; import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.indices.IndicesService; @@ -91,11 +93,11 @@ public class MetaDataMappingService extends AbstractComponent { static class UpdateTask extends MappingTask { final String type; - final CompressedString mappingSource; + final CompressedXContent mappingSource; final String nodeId; // null fr unknown final ActionListener listener; - UpdateTask(String index, String indexUUID, String type, CompressedString mappingSource, String nodeId, ActionListener listener) { + UpdateTask(String index, String indexUUID, String type, CompressedXContent mappingSource, String nodeId, ActionListener listener) { super(index, indexUUID); this.type = type; this.mappingSource = mappingSource; @@ -254,7 +256,7 @@ public class MetaDataMappingService extends AbstractComponent { UpdateTask updateTask = (UpdateTask) task; try { String type = updateTask.type; - CompressedString mappingSource = updateTask.mappingSource; + CompressedXContent mappingSource = updateTask.mappingSource; MappingMetaData mappingMetaData = builder.mapping(type); if (mappingMetaData != null && mappingMetaData.source().equals(mappingSource)) { @@ -376,9 +378,9 @@ public class MetaDataMappingService extends AbstractComponent { DocumentMapper existingMapper = indexService.mapperService().documentMapper(request.type()); if (MapperService.DEFAULT_MAPPING.equals(request.type())) { // _default_ types do not go through merging, but we do test the new settings. Also don't apply the old default - newMapper = indexService.mapperService().parse(request.type(), new CompressedString(request.source()), false); + newMapper = indexService.mapperService().parse(request.type(), new CompressedXContent(request.source()), false); } else { - newMapper = indexService.mapperService().parse(request.type(), new CompressedString(request.source()), existingMapper == null); + newMapper = indexService.mapperService().parse(request.type(), new CompressedXContent(request.source()), existingMapper == null); if (existingMapper != null) { // first, simulate MergeResult mergeResult = existingMapper.merge(newMapper.mapping(), true); @@ -386,9 +388,26 @@ public class MetaDataMappingService extends AbstractComponent { if (mergeResult.hasConflicts()) { throw new MergeMappingException(mergeResult.buildConflicts()); } + } else { + // TODO: can we find a better place for this validation? + // The reason this validation is here is that the mapper service doesn't learn about + // new types all at once , which can create a false error. + + // For example in MapperService we can't distinguish between a create index api call + // and a put mapping api call, so we don't which type did exist before. + // Also the order of the mappings may be backwards. + if (Version.indexCreated(indexService.getIndexSettings()).onOrAfter(Version.V_2_0_0) && newMapper.parentFieldMapper().active()) { + IndexMetaData indexMetaData = currentState.metaData().index(index); + for (ObjectCursor mapping : indexMetaData.mappings().values()) { + if (newMapper.parentFieldMapper().type().equals(mapping.value.type())) { + throw new IllegalArgumentException("can't add a _parent field that points to an already existing type"); + } + } + } } } + newMappers.put(index, newMapper); if (existingMapper != null) { existingMappers.put(index, existingMapper); @@ -415,12 +434,12 @@ public class MetaDataMappingService extends AbstractComponent { continue; } - CompressedString existingSource = null; + CompressedXContent existingSource = null; if (existingMappers.containsKey(entry.getKey())) { existingSource = existingMappers.get(entry.getKey()).mappingSource(); } DocumentMapper mergedMapper = indexService.mapperService().merge(newMapper.type(), newMapper.mappingSource(), false); - CompressedString updatedSource = mergedMapper.mappingSource(); + CompressedXContent updatedSource = mergedMapper.mappingSource(); if (existingSource != null) { if (existingSource.equals(updatedSource)) { diff --git a/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java index 52e28ffd21e..f4d81028448 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java @@ -202,9 +202,7 @@ public class RepositoriesMetaData extends AbstractDiffable implements Me builder.startObject(repository.name(), XContentBuilder.FieldCaseConversion.NONE); builder.field("type", repository.type()); builder.startObject("settings"); - for (Map.Entry settingEntry : repository.settings().getAsMap().entrySet()) { - builder.field(settingEntry.getKey(), settingEntry.getValue()); - } + repository.settings().toXContent(builder, params); builder.endObject(); builder.endObject(); diff --git a/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index 517a729afbf..f87a0521c83 100644 --- a/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -142,20 +142,20 @@ public class DiskThresholdDecider extends AllocationDecider { private void warnAboutDiskIfNeeded(DiskUsage usage) { // Check absolute disk values if (usage.getFreeBytes() < DiskThresholdDecider.this.freeBytesThresholdHigh.bytes()) { - logger.warn("high disk watermark [{} free] exceeded on {}, shards will be relocated away from this node", + logger.warn("high disk watermark [{}] exceeded on {}, shards will be relocated away from this node", DiskThresholdDecider.this.freeBytesThresholdHigh, usage); } else if (usage.getFreeBytes() < DiskThresholdDecider.this.freeBytesThresholdLow.bytes()) { - logger.info("low disk watermark [{} free] exceeded on {}, replicas will not be assigned to this node", + logger.info("low disk watermark [{}] exceeded on {}, replicas will not be assigned to this node", DiskThresholdDecider.this.freeBytesThresholdLow, usage); } // Check percentage disk values if (usage.getFreeDiskAsPercentage() < DiskThresholdDecider.this.freeDiskThresholdHigh) { - logger.warn("high disk watermark [{} free] exceeded on {}, shards will be relocated away from this node", - Strings.format1Decimals(DiskThresholdDecider.this.freeDiskThresholdHigh, "%"), usage); + logger.warn("high disk watermark [{}] exceeded on {}, shards will be relocated away from this node", + Strings.format1Decimals(100.0 - DiskThresholdDecider.this.freeDiskThresholdHigh, "%"), usage); } else if (usage.getFreeDiskAsPercentage() < DiskThresholdDecider.this.freeDiskThresholdLow) { - logger.info("low disk watermark [{} free] exceeded on {}, replicas will not be assigned to this node", - Strings.format1Decimals(DiskThresholdDecider.this.freeDiskThresholdLow, "%"), usage); + logger.info("low disk watermark [{}] exceeded on {}, replicas will not be assigned to this node", + Strings.format1Decimals(100.0 - DiskThresholdDecider.this.freeDiskThresholdLow, "%"), usage); } } @@ -234,6 +234,16 @@ public class DiskThresholdDecider extends AllocationDecider { return freeDiskThresholdHigh; } + // For Testing + public Double getUsedDiskThresholdLow() { + return 100.0 - freeDiskThresholdLow; + } + + // For Testing + public Double getUsedDiskThresholdHigh() { + return 100.0 - freeDiskThresholdHigh; + } + // For Testing public ByteSizeValue getFreeBytesThresholdLow() { return freeBytesThresholdLow; @@ -285,6 +295,8 @@ public class DiskThresholdDecider extends AllocationDecider { @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + double usedDiskThresholdLow = 100.0 - DiskThresholdDecider.this.freeDiskThresholdLow; + double usedDiskThresholdHigh = 100.0 - DiskThresholdDecider.this.freeDiskThresholdHigh; // Always allow allocation if the decider is disabled if (!enabled) { @@ -342,9 +354,11 @@ public class DiskThresholdDecider extends AllocationDecider { // First, check that the node currently over the low watermark double freeDiskPercentage = usage.getFreeDiskAsPercentage(); + // Cache the used disk percentage for displaying disk percentages consistent with documentation + double usedDiskPercentage = usage.getUsedDiskAsPercentage(); long freeBytes = usage.getFreeBytes(); if (logger.isTraceEnabled()) { - logger.trace("Node [{}] has {}% free disk", node.nodeId(), freeDiskPercentage); + logger.trace("Node [{}] has {}% used disk", node.nodeId(), usedDiskPercentage); } // a flag for whether the primary shard has been previously allocated @@ -387,20 +401,20 @@ public class DiskThresholdDecider extends AllocationDecider { // If the shard is a replica or has a primary that has already been allocated before, check the low threshold if (!shardRouting.primary() || (shardRouting.primary() && primaryHasBeenAllocated)) { if (logger.isDebugEnabled()) { - logger.debug("Less than the required {} free disk threshold ({} free) on node [{}], preventing allocation", - Strings.format1Decimals(freeDiskThresholdLow, "%"), - Strings.format1Decimals(freeDiskPercentage, "%"), node.nodeId()); + logger.debug("More than the allowed {} used disk threshold ({} used) on node [{}], preventing allocation", + Strings.format1Decimals(usedDiskThresholdLow, "%"), + Strings.format1Decimals(usedDiskPercentage, "%"), node.nodeId()); } - return allocation.decision(Decision.NO, NAME, "less than required [%s%%] free disk on node, free: [%s%%]", - freeDiskThresholdLow, freeDiskPercentage); + return allocation.decision(Decision.NO, NAME, "more than allowed [%s%%] used disk on node, free: [%s%%]", + usedDiskThresholdLow, freeDiskPercentage); } else if (freeDiskPercentage > freeDiskThresholdHigh) { // Allow the shard to be allocated because it is primary that // has never been allocated if it's under the high watermark if (logger.isDebugEnabled()) { - logger.debug("Less than the required {} free disk threshold ({} free) on node [{}], " + + logger.debug("More than the allowed {} used disk threshold ({} used) on node [{}], " + "but allowing allocation because primary has never been allocated", - Strings.format1Decimals(freeDiskThresholdLow, "%"), - Strings.format1Decimals(freeDiskPercentage, "%"), node.nodeId()); + Strings.format1Decimals(usedDiskThresholdLow, "%"), + Strings.format1Decimals(usedDiskPercentage, "%"), node.nodeId()); } return allocation.decision(Decision.YES, NAME, "primary has never been allocated before"); } else { @@ -412,8 +426,8 @@ public class DiskThresholdDecider extends AllocationDecider { Strings.format1Decimals(freeDiskThresholdHigh, "%"), Strings.format1Decimals(freeDiskPercentage, "%"), node.nodeId()); } - return allocation.decision(Decision.NO, NAME, "less than required [%s%%] free disk on node, free: [%s%%]", - freeDiskThresholdLow, freeDiskPercentage); + return allocation.decision(Decision.NO, NAME, "more than allowed [%s%%] used disk on node, free: [%s%%]", + usedDiskThresholdHigh, freeDiskPercentage); } } @@ -429,10 +443,10 @@ public class DiskThresholdDecider extends AllocationDecider { freeBytesThresholdLow, new ByteSizeValue(freeBytesAfterShard)); } if (freeSpaceAfterShard < freeDiskThresholdHigh) { - logger.warn("After allocating, node [{}] would have less than the required {} free disk threshold ({} free), preventing allocation", + logger.warn("After allocating, node [{}] would have more than the allowed {} free disk threshold ({} free), preventing allocation", node.nodeId(), Strings.format1Decimals(freeDiskThresholdHigh, "%"), Strings.format1Decimals(freeSpaceAfterShard, "%")); - return allocation.decision(Decision.NO, NAME, "after allocation less than required [%s%%] free disk on node, free: [%s%%]", - freeDiskThresholdLow, freeSpaceAfterShard); + return allocation.decision(Decision.NO, NAME, "after allocation more than allowed [%s%%] used disk on node, free: [%s%%]", + usedDiskThresholdLow, freeSpaceAfterShard); } return allocation.decision(Decision.YES, NAME, "enough disk for shard on node, free: [%s]", new ByteSizeValue(freeBytes)); diff --git a/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java index 1bc370cd894..add383b75fa 100644 --- a/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java +++ b/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java @@ -352,6 +352,7 @@ public class PagedBytesReference implements BytesReference { private final int offset; private final int length; private int pos; + private int mark; public PagedBytesReferenceStreamInput(ByteArray bytearray, int offset, int length) { this.bytearray = bytearray; @@ -420,9 +421,19 @@ public class PagedBytesReference implements BytesReference { return copiedBytes; } + @Override + public boolean markSupported() { + return true; + } + + @Override + public void mark(int readlimit) { + this.mark = pos; + } + @Override public void reset() throws IOException { - pos = 0; + pos = mark; } @Override diff --git a/src/main/java/org/elasticsearch/common/cli/CliTool.java b/src/main/java/org/elasticsearch/common/cli/CliTool.java index 71e87943349..b0784a57ca7 100644 --- a/src/main/java/org/elasticsearch/common/cli/CliTool.java +++ b/src/main/java/org/elasticsearch/common/cli/CliTool.java @@ -93,7 +93,7 @@ public abstract class CliTool { Preconditions.checkArgument(config.cmds().size() != 0, "At least one command must be configured"); this.config = config; this.terminal = terminal; - Tuple tuple = InternalSettingsPreparer.prepareSettings(EMPTY_SETTINGS, true); + Tuple tuple = InternalSettingsPreparer.prepareSettings(EMPTY_SETTINGS, true, terminal); settings = tuple.v1(); env = tuple.v2(); } diff --git a/src/main/java/org/elasticsearch/common/compress/CompressedIndexInput.java b/src/main/java/org/elasticsearch/common/compress/CompressedIndexInput.java index 06ec2a2f48f..12094108932 100644 --- a/src/main/java/org/elasticsearch/common/compress/CompressedIndexInput.java +++ b/src/main/java/org/elasticsearch/common/compress/CompressedIndexInput.java @@ -30,10 +30,9 @@ import java.io.IOException; * @deprecated Used only for backward comp. to read old compressed files, since we now use codec based compression */ @Deprecated -public abstract class CompressedIndexInput extends IndexInput { +public abstract class CompressedIndexInput extends IndexInput { private IndexInput in; - protected final T context; private int version; private long totalUncompressedLength; @@ -48,10 +47,9 @@ public abstract class CompressedIndexInput extends private int currentOffsetIdx; private long currentUncompressedChunkPointer; - public CompressedIndexInput(IndexInput in, T context) throws IOException { + public CompressedIndexInput(IndexInput in) throws IOException { super("compressed(" + in.toString() + ")"); this.in = in; - this.context = context; readHeader(in); this.version = in.readInt(); long metaDataPosition = in.readLong(); diff --git a/src/main/java/org/elasticsearch/common/compress/CompressedStreamInput.java b/src/main/java/org/elasticsearch/common/compress/CompressedStreamInput.java index 3df98a7f718..82eefe13a4c 100644 --- a/src/main/java/org/elasticsearch/common/compress/CompressedStreamInput.java +++ b/src/main/java/org/elasticsearch/common/compress/CompressedStreamInput.java @@ -27,10 +27,9 @@ import java.io.IOException; /** */ -public abstract class CompressedStreamInput extends StreamInput { +public abstract class CompressedStreamInput extends StreamInput { private final StreamInput in; - protected final CompressorContext context; private boolean closed; @@ -38,9 +37,8 @@ public abstract class CompressedStreamInput extends private int position = 0; private int valid = 0; - public CompressedStreamInput(StreamInput in, T context) throws IOException { + public CompressedStreamInput(StreamInput in) throws IOException { this.in = in; - this.context = context; super.setVersion(in.getVersion()); readHeader(in); } @@ -51,13 +49,6 @@ public abstract class CompressedStreamInput extends return super.setVersion(version); } - /** - * Expert!, resets to buffer start, without the need to decompress it again. - */ - public void resetToBufferStart() { - this.position = 0; - } - /** * Method is overridden to report number of bytes that can now be read * from decoded data buffer, without reading bytes from the underlying diff --git a/src/main/java/org/elasticsearch/common/compress/CompressedString.java b/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java similarity index 74% rename from src/main/java/org/elasticsearch/common/compress/CompressedString.java rename to src/main/java/org/elasticsearch/common/compress/CompressedXContent.java index aca1d45f86d..09ced0e29b2 100644 --- a/src/main/java/org/elasticsearch/common/compress/CompressedString.java +++ b/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java @@ -22,6 +22,7 @@ package org.elasticsearch.common.compress; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -34,33 +35,32 @@ import java.util.Arrays; * memory. Note that the compressed string might still sometimes need to be * decompressed in order to perform equality checks or to compute hash codes. */ -public final class CompressedString { +public final class CompressedXContent { private final byte[] bytes; private int hashCode; - public CompressedString(BytesReference data) throws IOException { + public CompressedXContent(BytesReference data) throws IOException { Compressor compressor = CompressorFactory.compressor(data); if (compressor != null) { // already compressed... this.bytes = data.toBytes(); } else { - BytesArray bytesArray = data.toBytesArray(); - this.bytes = CompressorFactory.defaultCompressor().compress(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length()); - assert CompressorFactory.compressor(bytes) != null; + BytesStreamOutput out = new BytesStreamOutput(); + try (StreamOutput compressedOutput = CompressorFactory.defaultCompressor().streamOutput(out)) { + data.writeTo(compressedOutput); + } + this.bytes = out.bytes().toBytes(); + assert CompressorFactory.compressor(new BytesArray(bytes)) != null; } } - public CompressedString(byte[] data, int offset, int length) throws IOException { - this(new BytesArray(data, offset, length)); + public CompressedXContent(byte[] data) throws IOException { + this(new BytesArray(data)); } - public CompressedString(byte[] data) throws IOException { - this(data, 0, data.length); - } - - public CompressedString(String str) throws IOException { + public CompressedXContent(String str) throws IOException { this(new BytesArray(new BytesRef(str))); } @@ -69,12 +69,15 @@ public final class CompressedString { return this.bytes; } + /** Return the compressed bytes as a {@link BytesReference}. */ + public BytesReference compressedReference() { + return new BytesArray(bytes); + } + /** Return the uncompressed bytes. */ public byte[] uncompressed() { - Compressor compressor = CompressorFactory.compressor(bytes); - assert compressor != null; try { - return compressor.uncompress(bytes, 0, bytes.length); + return CompressorFactory.uncompress(new BytesArray(bytes)).toBytes(); } catch (IOException e) { throw new IllegalStateException("Cannot decompress compressed string", e); } @@ -84,10 +87,10 @@ public final class CompressedString { return new BytesRef(uncompressed()).utf8ToString(); } - public static CompressedString readCompressedString(StreamInput in) throws IOException { + public static CompressedXContent readCompressedString(StreamInput in) throws IOException { byte[] bytes = new byte[in.readVInt()]; in.readBytes(bytes, 0, bytes.length); - return new CompressedString(bytes); + return new CompressedXContent(bytes); } public void writeTo(StreamOutput out) throws IOException { @@ -100,7 +103,7 @@ public final class CompressedString { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - CompressedString that = (CompressedString) o; + CompressedXContent that = (CompressedXContent) o; if (Arrays.equals(compressed(), that.compressed())) { return true; diff --git a/src/main/java/org/elasticsearch/common/compress/Compressor.java b/src/main/java/org/elasticsearch/common/compress/Compressor.java index 8d0199703ac..252fad09807 100644 --- a/src/main/java/org/elasticsearch/common/compress/Compressor.java +++ b/src/main/java/org/elasticsearch/common/compress/Compressor.java @@ -23,7 +23,6 @@ import org.apache.lucene.store.IndexInput; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.Settings; import org.jboss.netty.buffer.ChannelBuffer; import java.io.IOException; @@ -32,32 +31,20 @@ import java.io.IOException; */ public interface Compressor { - String type(); - - void configure(Settings settings); - boolean isCompressed(BytesReference bytes); - boolean isCompressed(byte[] data, int offset, int length); - boolean isCompressed(ChannelBuffer buffer); + StreamInput streamInput(StreamInput in) throws IOException; + + StreamOutput streamOutput(StreamOutput out) throws IOException; + + /** + * @deprecated Used for backward comp. since we now use Lucene compressed codec. + */ + @Deprecated boolean isCompressed(IndexInput in) throws IOException; - /** - * Uncompress the provided data, data can be detected as compressed using {@link #isCompressed(byte[], int, int)}. - */ - byte[] uncompress(byte[] data, int offset, int length) throws IOException; - - /** - * Compresses the provided data, data can be detected as compressed using {@link #isCompressed(byte[], int, int)}. - */ - byte[] compress(byte[] data, int offset, int length) throws IOException; - - CompressedStreamInput streamInput(StreamInput in) throws IOException; - - CompressedStreamOutput streamOutput(StreamOutput out) throws IOException; - /** * @deprecated Used for backward comp. since we now use Lucene compressed codec. */ diff --git a/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java b/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java index 9eb9c9d7212..72c57a97a01 100644 --- a/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java +++ b/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java @@ -19,68 +19,36 @@ package org.elasticsearch.common.compress; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import org.apache.lucene.store.IndexInput; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.compress.deflate.DeflateCompressor; import org.elasticsearch.common.compress.lzf.LZFCompressor; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.jboss.netty.buffer.ChannelBuffer; import java.io.IOException; -import java.util.List; -import java.util.Locale; /** */ public class CompressorFactory { - private static final LZFCompressor LZF = new LZFCompressor(); - private static final Compressor[] compressors; - private static final ImmutableMap compressorsByType; - private static Compressor defaultCompressor; + private static volatile Compressor defaultCompressor; static { - List compressorsX = Lists.newArrayList(); - compressorsX.add(LZF); - - compressors = compressorsX.toArray(new Compressor[compressorsX.size()]); - MapBuilder compressorsByTypeX = MapBuilder.newMapBuilder(); - for (Compressor compressor : compressors) { - compressorsByTypeX.put(compressor.type(), compressor); - } - compressorsByType = compressorsByTypeX.immutableMap(); - - defaultCompressor = LZF; + compressors = new Compressor[] { + new LZFCompressor(), + new DeflateCompressor() + }; + defaultCompressor = new DeflateCompressor(); } - public static synchronized void configure(Settings settings) { - for (Compressor compressor : compressors) { - compressor.configure(settings); - } - String defaultType = settings.get("compress.default.type", "lzf").toLowerCase(Locale.ENGLISH); - boolean found = false; - for (Compressor compressor : compressors) { - if (defaultType.equalsIgnoreCase(compressor.type())) { - defaultCompressor = compressor; - found = true; - break; - } - } - if (!found) { - Loggers.getLogger(CompressorFactory.class).warn("failed to find default type [{}]", defaultType); - } - } - - public static synchronized void setDefaultCompressor(Compressor defaultCompressor) { + public static void setDefaultCompressor(Compressor defaultCompressor) { CompressorFactory.defaultCompressor = defaultCompressor; } @@ -92,14 +60,10 @@ public class CompressorFactory { return compressor(bytes) != null; } - public static boolean isCompressed(byte[] data) { - return compressor(data, 0, data.length) != null; - } - - public static boolean isCompressed(byte[] data, int offset, int length) { - return compressor(data, offset, length) != null; - } - + /** + * @deprecated we don't compress lucene indexes anymore and rely on lucene codecs + */ + @Deprecated public static boolean isCompressed(IndexInput in) throws IOException { return compressor(in) != null; } @@ -108,37 +72,35 @@ public class CompressorFactory { public static Compressor compressor(BytesReference bytes) { for (Compressor compressor : compressors) { if (compressor.isCompressed(bytes)) { + // bytes should be either detected as compressed or as xcontent, + // if we have bytes that can be either detected as compressed or + // as a xcontent, we have a problem + assert XContentFactory.xContentType(bytes) == null; return compressor; } } - return null; - } - @Nullable - public static Compressor compressor(byte[] data) { - return compressor(data, 0, data.length); - } - - @Nullable - public static Compressor compressor(byte[] data, int offset, int length) { - for (Compressor compressor : compressors) { - if (compressor.isCompressed(data, offset, length)) { - return compressor; - } + XContentType contentType = XContentFactory.xContentType(bytes); + if (contentType == null) { + throw new NotXContentException("Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes"); } + return null; } - @Nullable public static Compressor compressor(ChannelBuffer buffer) { for (Compressor compressor : compressors) { if (compressor.isCompressed(buffer)) { return compressor; } } - return null; + throw new NotCompressedException(); } + /** + * @deprecated we don't compress lucene indexes anymore and rely on lucene codecs + */ + @Deprecated @Nullable public static Compressor compressor(IndexInput in) throws IOException { for (Compressor compressor : compressors) { @@ -149,25 +111,35 @@ public class CompressorFactory { return null; } - public static Compressor compressor(String type) { - return compressorsByType.get(type); - } - /** * Uncompress the provided data, data can be detected as compressed using {@link #isCompressed(byte[], int, int)}. */ public static BytesReference uncompressIfNeeded(BytesReference bytes) throws IOException { Compressor compressor = compressor(bytes); + BytesReference uncompressed; if (compressor != null) { - if (bytes.hasArray()) { - return new BytesArray(compressor.uncompress(bytes.array(), bytes.arrayOffset(), bytes.length())); - } - StreamInput compressed = compressor.streamInput(bytes.streamInput()); - BytesStreamOutput bStream = new BytesStreamOutput(); - Streams.copy(compressed, bStream); - compressed.close(); - return bStream.bytes(); + uncompressed = uncompress(bytes, compressor); + } else { + uncompressed = bytes; } - return bytes; + + return uncompressed; + } + + /** Decompress the provided {@link BytesReference}. */ + public static BytesReference uncompress(BytesReference bytes) throws IOException { + Compressor compressor = compressor(bytes); + if (compressor == null) { + throw new NotCompressedException(); + } + return uncompress(bytes, compressor); + } + + private static BytesReference uncompress(BytesReference bytes, Compressor compressor) throws IOException { + StreamInput compressed = compressor.streamInput(bytes.streamInput()); + BytesStreamOutput bStream = new BytesStreamOutput(); + Streams.copy(compressed, bStream); + compressed.close(); + return bStream.bytes(); } } diff --git a/src/main/java/org/elasticsearch/common/compress/CompressorContext.java b/src/main/java/org/elasticsearch/common/compress/NotCompressedException.java similarity index 73% rename from src/main/java/org/elasticsearch/common/compress/CompressorContext.java rename to src/main/java/org/elasticsearch/common/compress/NotCompressedException.java index 9ad70554046..653483fc586 100644 --- a/src/main/java/org/elasticsearch/common/compress/CompressorContext.java +++ b/src/main/java/org/elasticsearch/common/compress/NotCompressedException.java @@ -19,7 +19,13 @@ package org.elasticsearch.common.compress; -/** - */ -public interface CompressorContext { +/** Exception indicating that we were expecting something compressed, which + * was not compressed or corrupted so that the compression format could not + * be detected. */ +public class NotCompressedException extends RuntimeException { + + public NotCompressedException() { + super(); + } + } diff --git a/src/main/java/org/elasticsearch/common/compress/NotXContentException.java b/src/main/java/org/elasticsearch/common/compress/NotXContentException.java new file mode 100644 index 00000000000..68bbf4da81c --- /dev/null +++ b/src/main/java/org/elasticsearch/common/compress/NotXContentException.java @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.compress; + +import org.elasticsearch.common.xcontent.XContent; + +/** Exception indicating that we were expecting some {@link XContent} but could + * not detect its type. */ +public class NotXContentException extends RuntimeException { + + public NotXContentException(String message) { + super(message); + } + +} diff --git a/src/main/java/org/elasticsearch/common/compress/deflate/DeflateCompressor.java b/src/main/java/org/elasticsearch/common/compress/deflate/DeflateCompressor.java new file mode 100644 index 00000000000..b2aea1fa0ce --- /dev/null +++ b/src/main/java/org/elasticsearch/common/compress/deflate/DeflateCompressor.java @@ -0,0 +1,156 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.compress.deflate; + +import org.apache.lucene.store.IndexInput; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedIndexInput; +import org.elasticsearch.common.compress.Compressor; +import org.elasticsearch.common.io.stream.InputStreamStreamInput; +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.jboss.netty.buffer.ChannelBuffer; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; + +/** + * {@link Compressor} implementation based on the DEFLATE compression algorithm. + */ +public class DeflateCompressor implements Compressor { + + // An arbitrary header that we use to identify compressed streams + // It needs to be different from other compressors and to not be specific + // enough so that no stream starting with these bytes could be detected as + // a XContent + private static final byte[] HEADER = new byte[] { 'D', 'F', 'L', '\0' }; + // 3 is a good trade-off between speed and compression ratio + private static final int LEVEL = 3; + // We use buffering on the input and ouput of in/def-laters in order to + // limit the number of JNI calls + private static final int BUFFER_SIZE = 4096; + + @Override + public boolean isCompressed(BytesReference bytes) { + if (bytes.length() < HEADER.length) { + return false; + } + for (int i = 0; i < HEADER.length; ++i) { + if (bytes.get(i) != HEADER[i]) { + return false; + } + } + return true; + } + + @Override + public boolean isCompressed(ChannelBuffer buffer) { + if (buffer.readableBytes() < HEADER.length) { + return false; + } + final int offset = buffer.readerIndex(); + for (int i = 0; i < HEADER.length; ++i) { + if (buffer.getByte(offset + i) != HEADER[i]) { + return false; + } + } + return true; + } + + @Override + public StreamInput streamInput(StreamInput in) throws IOException { + final byte[] headerBytes = new byte[HEADER.length]; + int len = 0; + while (len < headerBytes.length) { + final int read = in.read(headerBytes, len, headerBytes.length - len); + if (read == -1) { + break; + } + len += read; + } + if (len != HEADER.length || Arrays.equals(headerBytes, HEADER) == false) { + throw new IllegalArgumentException("Input stream is not compressed with DEFLATE!"); + } + + final boolean nowrap = true; + final Inflater inflater = new Inflater(nowrap); + InputStream decompressedIn = new InflaterInputStream(in, inflater, BUFFER_SIZE); + decompressedIn = new BufferedInputStream(decompressedIn, BUFFER_SIZE); + return new InputStreamStreamInput(decompressedIn) { + private boolean closed = false; + + public void close() throws IOException { + try { + super.close(); + } finally { + if (closed == false) { + // important to release native memory + inflater.end(); + closed = true; + } + } + } + }; + } + + @Override + public StreamOutput streamOutput(StreamOutput out) throws IOException { + out.writeBytes(HEADER); + final boolean nowrap = true; + final Deflater deflater = new Deflater(LEVEL, nowrap); + final boolean syncFlush = true; + OutputStream compressedOut = new DeflaterOutputStream(out, deflater, BUFFER_SIZE, syncFlush); + compressedOut = new BufferedOutputStream(compressedOut, BUFFER_SIZE); + return new OutputStreamStreamOutput(compressedOut) { + private boolean closed = false; + + public void close() throws IOException { + try { + super.close(); + } finally { + if (closed == false) { + // important to release native memory + deflater.end(); + closed = true; + } + } + } + }; + } + + @Override + public boolean isCompressed(IndexInput in) throws IOException { + return false; + } + + @Override + public CompressedIndexInput indexInput(IndexInput in) throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java b/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java index 326eceb77c4..93bd583662b 100644 --- a/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java +++ b/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java @@ -32,14 +32,14 @@ import java.util.Arrays; /** */ @Deprecated -public class LZFCompressedIndexInput extends CompressedIndexInput { +public class LZFCompressedIndexInput extends CompressedIndexInput { private final ChunkDecoder decoder; // scratch area buffer private byte[] inputBuffer; public LZFCompressedIndexInput(IndexInput in, ChunkDecoder decoder) throws IOException { - super(in, LZFCompressorContext.INSTANCE); + super(in); this.decoder = decoder; this.uncompressed = new byte[LZFChunk.MAX_CHUNK_LEN]; diff --git a/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamInput.java b/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamInput.java index caaaadbeb3e..baefcaa8928 100644 --- a/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamInput.java +++ b/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamInput.java @@ -29,7 +29,7 @@ import java.io.IOException; /** */ -public class LZFCompressedStreamInput extends CompressedStreamInput { +public class LZFCompressedStreamInput extends CompressedStreamInput { private final BufferRecycler recycler; @@ -39,7 +39,7 @@ public class LZFCompressedStreamInput extends CompressedStreamInput= 3 && @@ -69,14 +58,6 @@ public class LZFCompressor implements Compressor { (bytes.get(2) == LZFChunk.BLOCK_TYPE_COMPRESSED || bytes.get(2) == LZFChunk.BLOCK_TYPE_NON_COMPRESSED); } - @Override - public boolean isCompressed(byte[] data, int offset, int length) { - return length >= 3 && - data[offset] == LZFChunk.BYTE_Z && - data[offset + 1] == LZFChunk.BYTE_V && - (data[offset + 2] == LZFChunk.BLOCK_TYPE_COMPRESSED || data[offset + 2] == LZFChunk.BLOCK_TYPE_NON_COMPRESSED); - } - @Override public boolean isCompressed(ChannelBuffer buffer) { int offset = buffer.readerIndex(); @@ -104,23 +85,13 @@ public class LZFCompressor implements Compressor { } @Override - public byte[] uncompress(byte[] data, int offset, int length) throws IOException { - return decoder.decode(data, offset, length); - } - - @Override - public byte[] compress(byte[] data, int offset, int length) throws IOException { - return LZFEncoder.safeEncode(data, offset, length); - } - - @Override - public CompressedStreamInput streamInput(StreamInput in) throws IOException { + public StreamInput streamInput(StreamInput in) throws IOException { return new LZFCompressedStreamInput(in, decoder); } @Override - public CompressedStreamOutput streamOutput(StreamOutput out) throws IOException { - return new LZFCompressedStreamOutput(out); + public StreamOutput streamOutput(StreamOutput out) throws IOException { + throw new UnsupportedOperationException("LZF is only here for back compat, no write support"); } @Override diff --git a/src/main/java/org/elasticsearch/common/io/PathUtils.java b/src/main/java/org/elasticsearch/common/io/PathUtils.java index 83ec5cf8214..103ac19df68 100644 --- a/src/main/java/org/elasticsearch/common/io/PathUtils.java +++ b/src/main/java/org/elasticsearch/common/io/PathUtils.java @@ -83,8 +83,9 @@ public final class PathUtils { */ public static Path get(Path[] roots, String path) { for (Path root : roots) { - Path normalizedPath = root.resolve(path).normalize(); - if(normalizedPath.startsWith(root)) { + Path normalizedRoot = root.normalize(); + Path normalizedPath = normalizedRoot.resolve(path).normalize(); + if(normalizedPath.startsWith(normalizedRoot)) { return normalizedPath; } } diff --git a/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java index ffe8d297ba4..e9aa52cf4d0 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java @@ -59,6 +59,16 @@ public class InputStreamStreamInput extends StreamInput { is.reset(); } + @Override + public boolean markSupported() { + return is.markSupported(); + } + + @Override + public void mark(int readlimit) { + is.mark(readlimit); + } + @Override public void close() throws IOException { is.close(); diff --git a/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java b/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java index 860588207f0..d1ee646fc34 100644 --- a/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java +++ b/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java @@ -24,11 +24,11 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.Scorer; import org.elasticsearch.script.ExplainableSearchScript; import org.elasticsearch.script.LeafSearchScript; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptException; import org.elasticsearch.script.SearchScript; import java.io.IOException; -import java.util.Map; public class ScriptScoreFunction extends ScoreFunction { @@ -71,17 +71,14 @@ public class ScriptScoreFunction extends ScoreFunction { } } - private final String sScript; - - private final Map params; + private final Script sScript; private final SearchScript script; - public ScriptScoreFunction(String sScript, Map params, SearchScript script) { + public ScriptScoreFunction(Script sScript, SearchScript script) { super(CombineFunction.REPLACE); this.sScript = sScript; - this.params = params; this.script = script; } @@ -114,8 +111,8 @@ public class ScriptScoreFunction extends ScoreFunction { } else { double score = score(docId, subQueryScore.getValue()); String explanation = "script score function, computed with script:\"" + sScript; - if (params != null) { - explanation += "\" and parameters: \n" + params.toString(); + if (sScript.getParams() != null) { + explanation += "\" and parameters: \n" + sScript.getParams().toString(); } Explanation scoreExp = Explanation.match( subQueryScore.getValue(), "_score: ", @@ -131,7 +128,7 @@ public class ScriptScoreFunction extends ScoreFunction { @Override public String toString() { - return "script[" + sScript + "], params [" + params + "]"; + return "script" + sScript.toString(); } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/common/settings/Settings.java b/src/main/java/org/elasticsearch/common/settings/Settings.java index 67bd3b00f44..7c79b3fc1d4 100644 --- a/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -1211,7 +1211,7 @@ public final class Settings implements ToXContent { * tries and resolve it against an environment variable ({@link System#getenv(String)}), and last, tries * and replace it with another setting already set on this builder. */ - public Builder replacePropertyPlaceholders() { + public Builder replacePropertyPlaceholders(String... ignoredValues) { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new PropertyPlaceholder.PlaceholderResolver() { @Override @@ -1241,7 +1241,19 @@ public final class Settings implements ToXContent { } }; for (Map.Entry entry : Maps.newHashMap(map).entrySet()) { - String value = propertyPlaceholder.replacePlaceholders(entry.getValue(), placeholderResolver); + String possiblePlaceholder = entry.getValue(); + boolean ignored = false; + for (String ignoredValue : ignoredValues) { + if (ignoredValue.equals(possiblePlaceholder)) { + ignored = true; + break; + } + } + if (ignored) { + continue; + } + + String value = propertyPlaceholder.replacePlaceholders(possiblePlaceholder, placeholderResolver); // if the values exists and has length, we should maintain it in the map // otherwise, the replace process resolved into removing it if (Strings.hasLength(value)) { diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java b/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java index 9ae1a03a67d..75e57509948 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java @@ -21,6 +21,7 @@ package org.elasticsearch.common.xcontent; import com.fasterxml.jackson.dataformat.cbor.CBORConstants; import com.fasterxml.jackson.dataformat.smile.SmileConstants; + import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -163,6 +164,9 @@ public class XContentFactory { if (c == '{') { return XContentType.JSON; } + if (Character.isWhitespace(c) == false) { + break; + } } return null; } @@ -204,65 +208,76 @@ public class XContentFactory { } /** - * Guesses the content type based on the provided input stream. + * Guesses the content type based on the provided input stream without consuming it. */ public static XContentType xContentType(InputStream si) throws IOException { - final int firstInt = si.read(); // this must be an int since we need to respect the method contract - if (firstInt == -1) { - return null; + if (si.markSupported() == false) { + throw new IllegalArgumentException("Cannot guess the xcontent type without mark/reset support on " + si.getClass()); } - - final int secondInt = si.read(); // this must be an int since we need to respect the method contract - if (secondInt == -1) { - return null; - } - final byte first = (byte) (0xff & firstInt); - final byte second = (byte) (0xff & secondInt); - if (first == SmileConstants.HEADER_BYTE_1 && second == SmileConstants.HEADER_BYTE_2) { - int third = si.read(); - if (third == SmileConstants.HEADER_BYTE_3) { - return XContentType.SMILE; - } - } - if (first == '{' || second == '{') { - return XContentType.JSON; - } - if (first == '-' && second == '-') { - int third = si.read(); - if (third == '-') { - return XContentType.YAML; - } - } - // CBOR logic similar to CBORFactory#hasCBORFormat - if (first == CBORConstants.BYTE_OBJECT_INDEFINITE){ - return XContentType.CBOR; - } - if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_TAG, first)) { - // Actually, specific "self-describe tag" is a very good indicator - int third = si.read(); - if (third == -1) { + si.mark(GUESS_HEADER_LENGTH); + try { + final int firstInt = si.read(); // this must be an int since we need to respect the method contract + if (firstInt == -1) { return null; } - if (first == (byte) 0xD9 && second == (byte) 0xD9 && third == (byte) 0xF7) { - return XContentType.CBOR; - } - } - // for small objects, some encoders just encode as major type object, we can safely - // say its CBOR since it doesn't contradict SMILE or JSON, and its a last resort - if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_OBJECT, first)) { - return XContentType.CBOR; - } - for (int i = 2; i < GUESS_HEADER_LENGTH; i++) { - int val = si.read(); - if (val == -1) { - return null; + final int secondInt = si.read(); // this must be an int since we need to respect the method contract + if (secondInt == -1) { + return null; } - if (val == '{') { + final byte first = (byte) (0xff & firstInt); + final byte second = (byte) (0xff & secondInt); + if (first == SmileConstants.HEADER_BYTE_1 && second == SmileConstants.HEADER_BYTE_2) { + int third = si.read(); + if (third == SmileConstants.HEADER_BYTE_3) { + return XContentType.SMILE; + } + } + if (first == '{' || second == '{') { return XContentType.JSON; } + if (first == '-' && second == '-') { + int third = si.read(); + if (third == '-') { + return XContentType.YAML; + } + } + // CBOR logic similar to CBORFactory#hasCBORFormat + if (first == CBORConstants.BYTE_OBJECT_INDEFINITE){ + return XContentType.CBOR; + } + if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_TAG, first)) { + // Actually, specific "self-describe tag" is a very good indicator + int third = si.read(); + if (third == -1) { + return null; + } + if (first == (byte) 0xD9 && second == (byte) 0xD9 && third == (byte) 0xF7) { + return XContentType.CBOR; + } + } + // for small objects, some encoders just encode as major type object, we can safely + // say its CBOR since it doesn't contradict SMILE or JSON, and its a last resort + if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_OBJECT, first)) { + return XContentType.CBOR; + } + + for (int i = 2; i < GUESS_HEADER_LENGTH; i++) { + int val = si.read(); + if (val == -1) { + return null; + } + if (val == '{') { + return XContentType.JSON; + } + if (Character.isWhitespace(val) == false) { + break; + } + } + return null; + } finally { + si.reset(); } - return null; } /** @@ -284,7 +299,7 @@ public class XContentFactory { * Guesses the content type based on the provided bytes. */ public static XContentType xContentType(BytesReference bytes) { - int length = bytes.length() < GUESS_HEADER_LENGTH ? bytes.length() : GUESS_HEADER_LENGTH; + int length = bytes.length(); if (length == 0) { return null; } @@ -316,9 +331,13 @@ public class XContentFactory { // a last chance for JSON for (int i = 0; i < length; i++) { - if (bytes.get(i) == '{') { + byte b = bytes.get(i); + if (b == '{') { return XContentType.JSON; } + if (Character.isWhitespace(b) == false) { + break; + } } return null; } diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index d196d459fbd..4efd18e8fa9 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -28,14 +28,14 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.compress.CompressedStreamInput; import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.ToXContent.Params; +import java.io.BufferedInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -49,45 +49,30 @@ import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; public class XContentHelper { public static XContentParser createParser(BytesReference bytes) throws IOException { - if (bytes.hasArray()) { - return createParser(bytes.array(), bytes.arrayOffset(), bytes.length()); - } Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { - CompressedStreamInput compressedInput = compressor.streamInput(bytes.streamInput()); + InputStream compressedInput = compressor.streamInput(bytes.streamInput()); + if (compressedInput.markSupported() == false) { + compressedInput = new BufferedInputStream(compressedInput); + } XContentType contentType = XContentFactory.xContentType(compressedInput); - compressedInput.resetToBufferStart(); return XContentFactory.xContent(contentType).createParser(compressedInput); } else { return XContentFactory.xContent(bytes).createParser(bytes.streamInput()); } } - - public static XContentParser createParser(byte[] data, int offset, int length) throws IOException { - Compressor compressor = CompressorFactory.compressor(data, offset, length); - if (compressor != null) { - CompressedStreamInput compressedInput = compressor.streamInput(StreamInput.wrap(data, offset, length)); - XContentType contentType = XContentFactory.xContentType(compressedInput); - compressedInput.resetToBufferStart(); - return XContentFactory.xContent(contentType).createParser(compressedInput); - } else { - return XContentFactory.xContent(data, offset, length).createParser(data, offset, length); - } - } - public static Tuple> convertToMap(BytesReference bytes, boolean ordered) throws ElasticsearchParseException { - if (bytes.hasArray()) { - return convertToMap(bytes.array(), bytes.arrayOffset(), bytes.length(), ordered); - } try { XContentParser parser; XContentType contentType; Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { - CompressedStreamInput compressedStreamInput = compressor.streamInput(bytes.streamInput()); + InputStream compressedStreamInput = compressor.streamInput(bytes.streamInput()); + if (compressedStreamInput.markSupported() == false) { + compressedStreamInput = new BufferedInputStream(compressedStreamInput); + } contentType = XContentFactory.xContentType(compressedStreamInput); - compressedStreamInput.resetToBufferStart(); parser = XContentFactory.xContent(contentType).createParser(compressedStreamInput); } else { contentType = XContentFactory.xContentType(bytes); @@ -103,34 +88,6 @@ public class XContentHelper { } } - public static Tuple> convertToMap(byte[] data, boolean ordered) throws ElasticsearchParseException { - return convertToMap(data, 0, data.length, ordered); - } - - public static Tuple> convertToMap(byte[] data, int offset, int length, boolean ordered) throws ElasticsearchParseException { - try { - XContentParser parser; - XContentType contentType; - Compressor compressor = CompressorFactory.compressor(data, offset, length); - if (compressor != null) { - CompressedStreamInput compressedStreamInput = compressor.streamInput(StreamInput.wrap(data, offset, length)); - contentType = XContentFactory.xContentType(compressedStreamInput); - compressedStreamInput.resetToBufferStart(); - parser = XContentFactory.xContent(contentType).createParser(compressedStreamInput); - } else { - contentType = XContentFactory.xContentType(data, offset, length); - parser = XContentFactory.xContent(contentType).createParser(data, offset, length); - } - if (ordered) { - return Tuple.tuple(contentType, parser.mapOrderedAndClose()); - } else { - return Tuple.tuple(contentType, parser.mapAndClose()); - } - } catch (IOException e) { - throw new ElasticsearchParseException("Failed to parse content to map", e); - } - } - public static String convertToJson(BytesReference bytes, boolean reformatJson) throws IOException { return convertToJson(bytes, reformatJson, false); } @@ -426,9 +383,11 @@ public class XContentHelper { public static void writeDirect(BytesReference source, XContentBuilder rawBuilder, ToXContent.Params params) throws IOException { Compressor compressor = CompressorFactory.compressor(source); if (compressor != null) { - CompressedStreamInput compressedStreamInput = compressor.streamInput(source.streamInput()); + InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); + if (compressedStreamInput.markSupported() == false) { + compressedStreamInput = new BufferedInputStream(compressedStreamInput); + } XContentType contentType = XContentFactory.xContentType(compressedStreamInput); - compressedStreamInput.resetToBufferStart(); if (contentType == rawBuilder.contentType()) { Streams.copy(compressedStreamInput, rawBuilder.stream()); } else { @@ -457,9 +416,11 @@ public class XContentHelper { public static void writeRawField(String field, BytesReference source, XContentBuilder builder, ToXContent.Params params) throws IOException { Compressor compressor = CompressorFactory.compressor(source); if (compressor != null) { - CompressedStreamInput compressedStreamInput = compressor.streamInput(source.streamInput()); + InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); + if (compressedStreamInput.markSupported() == false) { + compressedStreamInput = new BufferedInputStream(compressedStreamInput); + } XContentType contentType = XContentFactory.xContentType(compressedStreamInput); - compressedStreamInput.resetToBufferStart(); if (contentType == builder.contentType()) { builder.rawField(field, compressedStreamInput); } else { diff --git a/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index 4acee241603..329bad87265 100644 --- a/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -19,11 +19,15 @@ package org.elasticsearch.common.xcontent; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.cbor.CborXContent; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.smile.SmileXContent; import org.elasticsearch.common.xcontent.yaml.YamlXContent; +import java.io.IOException; + /** * The content type of {@link org.elasticsearch.common.xcontent.XContent}. */ @@ -144,4 +148,18 @@ public enum XContentType { public abstract String shortName(); public abstract XContent xContent(); + + public static XContentType readFrom(StreamInput in) throws IOException { + int index = in.readVInt(); + for (XContentType contentType : values()) { + if (index == contentType.index) { + return contentType; + } + } + throw new IllegalStateException("Unknown XContentType with index [" + index + "]"); + } + + public static void writeTo(XContentType contentType, StreamOutput out) throws IOException { + out.writeVInt(contentType.index); + } } diff --git a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java index 92d5bad4bf6..7fd585a6a41 100644 --- a/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java +++ b/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java @@ -227,21 +227,21 @@ public class PublishClusterStateAction extends AbstractComponent { public static BytesReference serializeFullClusterState(ClusterState clusterState, Version nodeVersion) throws IOException { BytesStreamOutput bStream = new BytesStreamOutput(); - StreamOutput stream = CompressorFactory.defaultCompressor().streamOutput(bStream); - stream.setVersion(nodeVersion); - stream.writeBoolean(true); - clusterState.writeTo(stream); - stream.close(); + try (StreamOutput stream = CompressorFactory.defaultCompressor().streamOutput(bStream)) { + stream.setVersion(nodeVersion); + stream.writeBoolean(true); + clusterState.writeTo(stream); + } return bStream.bytes(); } public static BytesReference serializeDiffClusterState(Diff diff, Version nodeVersion) throws IOException { BytesStreamOutput bStream = new BytesStreamOutput(); - StreamOutput stream = CompressorFactory.defaultCompressor().streamOutput(bStream); - stream.setVersion(nodeVersion); - stream.writeBoolean(false); - diff.writeTo(stream); - stream.close(); + try (StreamOutput stream = CompressorFactory.defaultCompressor().streamOutput(bStream)) { + stream.setVersion(nodeVersion); + stream.writeBoolean(false); + diff.writeTo(stream); + } return bStream.bytes(); } diff --git a/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java index f4385947dc8..e0e7a847d45 100644 --- a/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java +++ b/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java @@ -56,6 +56,7 @@ import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData; import java.util.*; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; /** * @@ -166,12 +167,12 @@ public class GatewayAllocator extends AbstractComponent { AsyncShardFetch fetch = asyncFetchStarted.get(shard.shardId()); if (fetch == null) { - fetch = new InternalAsyncFetch<>(logger, "shard_started", shard.shardId(), startedAction, clusterService, allocationService); + fetch = new InternalAsyncFetch<>(logger, "shard_started", shard.shardId(), startedAction); asyncFetchStarted.put(shard.shardId(), fetch); } AsyncShardFetch.FetchResult shardState = fetch.fetchData(nodes, metaData, allocation.getIgnoreNodes(shard.shardId())); if (shardState.hasData() == false) { - logger.trace("{}: ignoring allocation, still fetching shard started state"); + logger.trace("{}: ignoring allocation, still fetching shard started state", shard); unassignedIterator.remove(); routingNodes.ignoredUnassigned().add(shard); continue; @@ -395,7 +396,7 @@ public class GatewayAllocator extends AbstractComponent { } if (!canBeAllocatedToAtLeastOneNode) { - logger.trace("{}: ignoring allocation, can't be allocated on any node"); + logger.trace("{}: ignoring allocation, can't be allocated on any node", shard); unassignedIterator.remove(); routingNodes.ignoredUnassigned().add(shard); continue; @@ -403,12 +404,12 @@ public class GatewayAllocator extends AbstractComponent { AsyncShardFetch fetch = asyncFetchStore.get(shard.shardId()); if (fetch == null) { - fetch = new InternalAsyncFetch<>(logger, "shard_store", shard.shardId(), storeAction, clusterService, allocationService); + fetch = new InternalAsyncFetch<>(logger, "shard_store", shard.shardId(), storeAction); asyncFetchStore.put(shard.shardId(), fetch); } AsyncShardFetch.FetchResult shardStores = fetch.fetchData(nodes, metaData, allocation.getIgnoreNodes(shard.shardId())); if (shardStores.hasData() == false) { - logger.trace("{}: ignoring allocation, still fetching shard stores"); + logger.trace("{}: ignoring allocation, still fetching shard stores", shard); unassignedIterator.remove(); routingNodes.ignoredUnassigned().add(shard); continue; // still fetching @@ -513,23 +514,24 @@ public class GatewayAllocator extends AbstractComponent { return changed; } - static class InternalAsyncFetch extends AsyncShardFetch { + private final AtomicBoolean rerouting = new AtomicBoolean(); - private final ClusterService clusterService; - private final AllocationService allocationService; + class InternalAsyncFetch extends AsyncShardFetch { - public InternalAsyncFetch(ESLogger logger, String type, ShardId shardId, List, T> action, - ClusterService clusterService, AllocationService allocationService) { + public InternalAsyncFetch(ESLogger logger, String type, ShardId shardId, List, T> action) { super(logger, type, shardId, action); - this.clusterService = clusterService; - this.allocationService = allocationService; } @Override protected void reroute(ShardId shardId, String reason) { - clusterService.submitStateUpdateTask("async_shard_fetch(" + type + ") " + shardId + ", reasons (" + reason + ")", Priority.HIGH, new ClusterStateUpdateTask() { + if (rerouting.compareAndSet(false, true) == false) { + logger.trace("{} already has pending reroute, ignoring {}", shardId, reason); + return; + } + clusterService.submitStateUpdateTask("async_shard_fetch", Priority.HIGH, new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) throws Exception { + rerouting.set(false); if (currentState.nodes().masterNode() == null) { return currentState; } diff --git a/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index 9d88d84f64a..9ea7cf5e60b 100644 --- a/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -21,16 +21,26 @@ package org.elasticsearch.gateway; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.Collections2; + import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.store.*; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.OutputStreamIndexOutput; +import org.apache.lucene.store.SimpleFSDirectory; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.store.InputStreamIndexInput; -import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.io.OutputStream; @@ -280,7 +290,7 @@ public abstract class MetaDataStateFormat { logger.debug("{}: no data for [{}], ignoring...", prefix, stateFile.toAbsolutePath()); continue; } - parser = XContentHelper.createParser(data, 0, data.length); + parser = XContentHelper.createParser(new BytesArray(data)); state = fromXContent(parser); if (state == null) { logger.debug("{}: no data for [{}], ignoring...", prefix, stateFile.toAbsolutePath()); diff --git a/src/main/java/org/elasticsearch/index/AbstractIndexComponent.java b/src/main/java/org/elasticsearch/index/AbstractIndexComponent.java index 3cfecc4cf07..b229b365665 100644 --- a/src/main/java/org/elasticsearch/index/AbstractIndexComponent.java +++ b/src/main/java/org/elasticsearch/index/AbstractIndexComponent.java @@ -53,6 +53,10 @@ public abstract class AbstractIndexComponent implements IndexComponent { return this.index; } + public Settings indexSettings() { + return indexSettings; + } + public String nodeName() { return indexSettings.get("name", ""); } diff --git a/src/main/java/org/elasticsearch/index/IndexService.java b/src/main/java/org/elasticsearch/index/IndexService.java index 09335126c73..e6ff7f232f2 100644 --- a/src/main/java/org/elasticsearch/index/IndexService.java +++ b/src/main/java/org/elasticsearch/index/IndexService.java @@ -36,36 +36,37 @@ import org.elasticsearch.index.aliases.IndexAliasesService; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.cache.IndexCache; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.cache.bitset.ShardBitsetFilterCacheModule; +import org.elasticsearch.index.cache.bitset.ShardBitsetFilterCache; import org.elasticsearch.index.cache.filter.ShardFilterCache; -import org.elasticsearch.index.cache.filter.ShardFilterCacheModule; -import org.elasticsearch.index.cache.query.ShardQueryCacheModule; +import org.elasticsearch.index.cache.query.ShardQueryCache; import org.elasticsearch.index.deletionpolicy.DeletionPolicyModule; import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.fielddata.ShardFieldDataModule; -import org.elasticsearch.index.gateway.IndexShardGatewayModule; +import org.elasticsearch.index.fielddata.ShardFieldData; +import org.elasticsearch.index.gateway.IndexShardGateway; import org.elasticsearch.index.gateway.IndexShardGatewayService; -import org.elasticsearch.index.get.ShardGetModule; -import org.elasticsearch.index.indexing.ShardIndexingModule; +import org.elasticsearch.index.get.ShardGetService; +import org.elasticsearch.index.indexing.ShardIndexingService; +import org.elasticsearch.index.indexing.slowlog.ShardSlowLogIndexingService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.merge.policy.MergePolicyModule; import org.elasticsearch.index.merge.policy.MergePolicyProvider; import org.elasticsearch.index.merge.scheduler.MergeSchedulerModule; import org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider; import org.elasticsearch.index.percolator.PercolatorQueriesRegistry; -import org.elasticsearch.index.percolator.PercolatorShardModule; +import org.elasticsearch.index.percolator.stats.ShardPercolateService; import org.elasticsearch.index.query.IndexQueryParserService; -import org.elasticsearch.index.search.stats.ShardSearchModule; +import org.elasticsearch.index.search.slowlog.ShardSlowLogSearchService; +import org.elasticsearch.index.search.stats.ShardSearchService; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.settings.IndexSettingsService; import org.elasticsearch.index.shard.*; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.index.snapshots.IndexShardSnapshotModule; +import org.elasticsearch.index.snapshots.IndexShardSnapshotAndRestoreService; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.store.StoreModule; -import org.elasticsearch.index.suggest.SuggestShardModule; -import org.elasticsearch.index.termvectors.ShardTermVectorsModule; +import org.elasticsearch.index.suggest.stats.ShardSuggestService; +import org.elasticsearch.index.termvectors.ShardTermVectorsService; import org.elasticsearch.index.translog.TranslogService; import org.elasticsearch.indices.IndicesLifecycle; import org.elasticsearch.indices.IndicesService; @@ -307,24 +308,12 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone final ShardFilterCache shardFilterCache = new ShardFilterCache(shardId, injector.getInstance(IndicesFilterCache.class)); ModulesBuilder modules = new ModulesBuilder(); modules.add(new ShardsPluginsModule(indexSettings, pluginsService)); - modules.add(new IndexShardModule(shardId, primary, indexSettings)); - modules.add(new ShardIndexingModule()); - modules.add(new ShardSearchModule()); - modules.add(new ShardGetModule()); + modules.add(new IndexShardModule(shardId, primary, indexSettings, shardFilterCache)); modules.add(new StoreModule(injector.getInstance(IndexStore.class).shardDirectory(), lock, new StoreCloseListener(shardId, canDeleteShardContent, shardFilterCache), path)); modules.add(new DeletionPolicyModule(indexSettings)); modules.add(new MergePolicyModule(indexSettings)); modules.add(new MergeSchedulerModule(indexSettings)); - modules.add(new ShardFilterCacheModule(shardFilterCache)); - modules.add(new ShardQueryCacheModule()); - modules.add(new ShardBitsetFilterCacheModule()); - modules.add(new ShardFieldDataModule()); - modules.add(new IndexShardGatewayModule()); - modules.add(new PercolatorShardModule()); - modules.add(new ShardTermVectorsModule()); - modules.add(new IndexShardSnapshotModule()); - modules.add(new SuggestShardModule()); try { shardInjector = modules.createChildInjector(injector); } catch (CreationException e) { diff --git a/src/main/java/org/elasticsearch/index/aliases/IndexAlias.java b/src/main/java/org/elasticsearch/index/aliases/IndexAlias.java index 3d02731dbfa..48ebc4239ac 100644 --- a/src/main/java/org/elasticsearch/index/aliases/IndexAlias.java +++ b/src/main/java/org/elasticsearch/index/aliases/IndexAlias.java @@ -21,7 +21,7 @@ package org.elasticsearch.index.aliases; import org.apache.lucene.search.Query; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; /** * @@ -30,11 +30,11 @@ public class IndexAlias { private final String alias; - private final CompressedString filter; + private final CompressedXContent filter; private final Query parsedFilter; - public IndexAlias(String alias, @Nullable CompressedString filter, @Nullable Query parsedFilter) { + public IndexAlias(String alias, @Nullable CompressedXContent filter, @Nullable Query parsedFilter) { this.alias = alias; this.filter = filter; this.parsedFilter = parsedFilter; @@ -45,7 +45,7 @@ public class IndexAlias { } @Nullable - public CompressedString filter() { + public CompressedXContent filter() { return filter; } diff --git a/src/main/java/org/elasticsearch/index/aliases/IndexAliasesService.java b/src/main/java/org/elasticsearch/index/aliases/IndexAliasesService.java index a097a01675b..21d6582e03f 100644 --- a/src/main/java/org/elasticsearch/index/aliases/IndexAliasesService.java +++ b/src/main/java/org/elasticsearch/index/aliases/IndexAliasesService.java @@ -23,7 +23,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; @@ -63,11 +63,11 @@ public class IndexAliasesService extends AbstractIndexComponent implements Itera return aliases.get(alias); } - public IndexAlias create(String alias, @Nullable CompressedString filter) { + public IndexAlias create(String alias, @Nullable CompressedXContent filter) { return new IndexAlias(alias, filter, parse(alias, filter)); } - public void add(String alias, @Nullable CompressedString filter) { + public void add(String alias, @Nullable CompressedXContent filter) { add(new IndexAlias(alias, filter, parse(alias, filter))); } @@ -120,7 +120,7 @@ public class IndexAliasesService extends AbstractIndexComponent implements Itera aliases.remove(alias); } - private Query parse(String alias, CompressedString filter) { + private Query parse(String alias, CompressedXContent filter) { if (filter == null) { return null; } diff --git a/src/main/java/org/elasticsearch/index/cache/filter/ShardFilterCacheModule.java b/src/main/java/org/elasticsearch/index/cache/filter/ShardFilterCacheModule.java deleted file mode 100644 index 37bcb805768..00000000000 --- a/src/main/java/org/elasticsearch/index/cache/filter/ShardFilterCacheModule.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache.filter; - -import org.elasticsearch.common.inject.AbstractModule; - -/** - */ -public class ShardFilterCacheModule extends AbstractModule { - - private final ShardFilterCache shardFilterCache; - - public ShardFilterCacheModule(ShardFilterCache shardFilterCache) { - this.shardFilterCache = shardFilterCache; - } - - @Override - protected void configure() { - bind(ShardFilterCache.class).toInstance(shardFilterCache); - } -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java index e748805e329..6f78ec5fd53 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java +++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java @@ -21,16 +21,10 @@ package org.elasticsearch.index.fielddata.plain; import com.carrotsearch.hppc.ObjectObjectHashMap; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.*; import org.apache.lucene.index.MultiDocValues.OrdinalMap; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; @@ -39,6 +33,7 @@ import org.apache.lucene.util.PagedBytes; import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -47,14 +42,8 @@ import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; -import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; -import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; -import org.elasticsearch.index.fielddata.FieldDataType; -import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.*; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.IndexParentChildFieldData; -import org.elasticsearch.index.fielddata.RamAccountingTermsEnum; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.ordinals.Ordinals; import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder; @@ -70,17 +59,7 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.MultiValueMode; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.NavigableSet; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; import java.util.concurrent.TimeUnit; /** @@ -89,7 +68,7 @@ import java.util.concurrent.TimeUnit; */ public class ParentChildIndexFieldData extends AbstractIndexFieldData implements IndexParentChildFieldData, DocumentTypeListener { - private final NavigableSet parentTypes; + private final NavigableSet parentTypes; private final CircuitBreakerService breakerService; // If child type (a type with _parent field) is added or removed, we want to make sure modifications don't happen @@ -100,7 +79,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData(BytesRef.getUTF8SortedAsUnicodeComparator()); + parentTypes = new TreeSet<>(); this.breakerService = breakerService; for (DocumentMapper documentMapper : mapperService.docMappers(false)) { beforeCreate(documentMapper); @@ -114,15 +93,60 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData parentTypes; + synchronized (lock) { + parentTypes = ImmutableSortedSet.copyOf(this.parentTypes); + } + return new AbstractAtomicParentChildFieldData() { + + public Set types() { + return parentTypes; + } + + @Override + public SortedDocValues getOrdinalsValues(String type) { + try { + return DocValues.getSorted(reader, ParentFieldMapper.joinField(type)); + } catch (IOException e) { + throw new IllegalStateException("cannot load join doc values field for type [" + type + "]", e); + } + } + + @Override + public long ramBytesUsed() { + // unknown + return 0; + } + + @Override + public Collection getChildResources() { + return Collections.emptyList(); + } + + @Override + public void close() throws ElasticsearchException { + } + }; + } else { + return super.load(context); + } + } + + @Override + public AbstractAtomicParentChildFieldData loadDirect(LeafReaderContext context) throws Exception { LeafReader reader = context.reader(); final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat( "acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO ); - final NavigableSet parentTypes; + final NavigableSet parentTypes = new TreeSet<>(); synchronized (lock) { - parentTypes = ImmutableSortedSet.copyOf(BytesRef.getUTF8SortedAsUnicodeComparator(), this.parentTypes); + for (String parentType : this.parentTypes) { + parentTypes.add(new BytesRef(parentType)); + } } boolean success = false; ParentChildAtomicFieldData data = null; @@ -192,7 +216,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData parentTypes = new HashSet<>(); + final Set parentTypes; synchronized (lock) { - for (BytesRef type : this.parentTypes) { - parentTypes.add(type.utf8ToString()); - } + parentTypes = ImmutableSet.copyOf(this.parentTypes); } long ramBytesUsed = 0; @@ -352,7 +374,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData ordinalMapPerType; - GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed) { + GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed, Map ordinalMapPerType) { this.reader = reader; this.ramBytesUsed = ramBytesUsed; this.fielddata = fielddata; + this.ordinalMapPerType = ordinalMapPerType; } @Override @@ -514,4 +538,20 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData parameters) { - sourceTransforms.add(new ScriptTransform(scriptService, script, scriptType, language, parameters)); + public Builder transform(ScriptService scriptService, Script script) { + sourceTransforms.add(new ScriptTransform(scriptService, script)); + return this; + } + + /** + * @deprecated Use {@link #transform(ScriptService, Script)} instead. + */ + @Deprecated + public Builder transform(ScriptService scriptService, String script, ScriptType scriptType, String language, + Map parameters) { + sourceTransforms.add(new ScriptTransform(scriptService, new Script(script, scriptType, language, parameters))); return this; } @@ -150,7 +160,7 @@ public class DocumentMapper implements ToXContent { private final String type; private final StringAndBytesText typeText; - private volatile CompressedString mappingSource; + private volatile CompressedXContent mappingSource; private final Mapping mapping; @@ -235,7 +245,7 @@ public class DocumentMapper implements ToXContent { return mapping.meta; } - public CompressedString mappingSource() { + public CompressedXContent mappingSource() { return this.mappingSource; } @@ -388,20 +398,24 @@ public class DocumentMapper implements ToXContent { private void addFieldMappers(Collection fieldMappers) { assert mappingLock.isWriteLockedByCurrentThread(); - this.fieldMappers = this.fieldMappers.copyAndAllAll(fieldMappers); + this.fieldMappers = this.fieldMappers.copyAndAllAll(fieldMappers); mapperService.addFieldMappers(fieldMappers); } + public boolean isParent(String type) { + return mapperService.getParentTypes().contains(type); + } + private void addObjectMappers(Collection objectMappers) { assert mappingLock.isWriteLockedByCurrentThread(); - MapBuilder builder = MapBuilder.newMapBuilder(this.objectMappers); - for (ObjectMapper objectMapper : objectMappers) { - builder.put(objectMapper.fullPath(), objectMapper); - if (objectMapper.nested().isNested()) { - hasNestedObjects = true; + MapBuilder builder = MapBuilder.newMapBuilder(this.objectMappers); + for (ObjectMapper objectMapper : objectMappers) { + builder.put(objectMapper.fullPath(), objectMapper); + if (objectMapper.nested().isNested()) { + hasNestedObjects = true; + } } - } - this.objectMappers = builder.immutableMap(); + this.objectMappers = builder.immutableMap(); mapperService.addObjectMappers(objectMappers); } @@ -454,26 +468,26 @@ public class DocumentMapper implements ToXContent { public MergeResult merge(Mapping mapping, boolean simulate) { try (ReleasableLock lock = mappingWriteLock.acquire()) { - final MergeResult mergeResult = newMergeContext(simulate); - this.mapping.merge(mapping, mergeResult); - if (simulate == false) { - addFieldMappers(mergeResult.getNewFieldMappers()); - addObjectMappers(mergeResult.getNewObjectMappers()); - refreshSource(); - } - return mergeResult; + final MergeResult mergeResult = newMergeContext(simulate); + this.mapping.merge(mapping, mergeResult); + if (simulate == false) { + addFieldMappers(mergeResult.getNewFieldMappers()); + addObjectMappers(mergeResult.getNewObjectMappers()); + refreshSource(); } + return mergeResult; + } } private void refreshSource() throws ElasticsearchGenerationException { try { BytesStreamOutput bStream = new BytesStreamOutput(); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, CompressorFactory.defaultCompressor().streamOutput(bStream)); - builder.startObject(); - toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - builder.close(); - mappingSource = new CompressedString(bStream.bytes()); + try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, CompressorFactory.defaultCompressor().streamOutput(bStream))) { + builder.startObject(); + toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + } + mappingSource = new CompressedXContent(bStream.bytes()); } catch (Exception e) { throw new ElasticsearchGenerationException("failed to serialize source for type [" + type + "]", e); } @@ -498,28 +512,13 @@ public class DocumentMapper implements ToXContent { private static class ScriptTransform implements SourceTransform { private final ScriptService scriptService; /** - * Contents of the script to transform the source document before indexing. + * The script to transform the source document before indexing. */ - private final String script; - /** - * The type of the script to run. - */ - private final ScriptType scriptType; - /** - * Language of the script to transform the source document before indexing. - */ - private final String language; - /** - * Parameters passed to the transform script. - */ - private final Map parameters; + private final Script script; - public ScriptTransform(ScriptService scriptService, String script, ScriptType scriptType, String language, Map parameters) { + public ScriptTransform(ScriptService scriptService, Script script) { this.scriptService = scriptService; this.script = script; - this.scriptType = scriptType; - this.language = language; - this.parameters = parameters; } @Override @@ -527,7 +526,7 @@ public class DocumentMapper implements ToXContent { public Map transformSourceAsMap(Map sourceAsMap) { try { // We use the ctx variable and the _source name to be consistent with the update api. - ExecutableScript executable = scriptService.executable(new Script(language, script, scriptType, parameters), ScriptContext.Standard.MAPPING); + ExecutableScript executable = scriptService.executable(script, ScriptContext.Standard.MAPPING); Map ctx = new HashMap<>(1); ctx.put("_source", sourceAsMap); executable.setNextVar("ctx", ctx); @@ -541,16 +540,7 @@ public class DocumentMapper implements ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("script", script); - if (language != null) { - builder.field("lang", language); - } - if (parameters != null) { - builder.field("params", parameters); - } - builder.endObject(); - return builder; + return script.toXContent(builder, params); } } } diff --git a/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 9084e17d60b..d5a3ff1f9ad 100644 --- a/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -27,7 +27,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; @@ -71,10 +71,8 @@ import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.similarity.SimilarityLookupService; -import org.elasticsearch.script.ScriptParameterParser; -import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptService.ScriptType; import java.util.Iterator; import java.util.List; @@ -194,15 +192,15 @@ public class DocumentMapperParser extends AbstractIndexComponent { return parse(type, mapping, defaultSource); } - public DocumentMapper parseCompressed(@Nullable String type, CompressedString source) throws MapperParsingException { + public DocumentMapper parseCompressed(@Nullable String type, CompressedXContent source) throws MapperParsingException { return parseCompressed(type, source, null); } @SuppressWarnings({"unchecked"}) - public DocumentMapper parseCompressed(@Nullable String type, CompressedString source, String defaultSource) throws MapperParsingException { + public DocumentMapper parseCompressed(@Nullable String type, CompressedXContent source, String defaultSource) throws MapperParsingException { Map mapping = null; if (source != null) { - Map root = XContentHelper.convertToMap(source.compressed(), true).v2(); + Map root = XContentHelper.convertToMap(source.compressedReference(), true).v2(); Tuple> t = extractMapping(type, root); type = t.v1(); mapping = t.v2(); @@ -238,7 +236,6 @@ public class DocumentMapperParser extends AbstractIndexComponent { Object fieldNode = entry.getValue(); if ("transform".equals(fieldName)) { - iterator.remove(); if (fieldNode instanceof Map) { parseTransform(docBuilder, (Map) fieldNode, parserContext.indexVersionCreated()); } else if (fieldNode instanceof List) { @@ -251,6 +248,7 @@ public class DocumentMapperParser extends AbstractIndexComponent { } else { throw new MapperParsingException("Transform must be an object or an array but was: " + fieldNode); } + iterator.remove(); } else { Mapper.TypeParser typeParser = rootTypeParsers.get(fieldName); if (typeParser != null) { @@ -296,23 +294,10 @@ public class DocumentMapperParser extends AbstractIndexComponent { return remainingFields.toString(); } - @SuppressWarnings("unchecked") private void parseTransform(DocumentMapper.Builder docBuilder, Map transformConfig, Version indexVersionCreated) { - ScriptParameterParser scriptParameterParser = new ScriptParameterParser(); - scriptParameterParser.parseConfig(transformConfig, true); - - String script = null; - ScriptType scriptType = null; - ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue(); - if (scriptValue != null) { - script = scriptValue.script(); - scriptType = scriptValue.scriptType(); - } - + Script script = Script.parse(transformConfig, true); if (script != null) { - String scriptLang = scriptParameterParser.lang(); - Map params = (Map)transformConfig.remove("params"); - docBuilder.transform(scriptService, script, scriptType, scriptLang, params); + docBuilder.transform(scriptService, script); } checkNoRemainingFields(transformConfig, indexVersionCreated, "Transform config has unsupported parameters: "); } diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/src/main/java/org/elasticsearch/index/mapper/MapperService.java index b63df2d6cc4..d592231d20e 100755 --- a/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -21,10 +21,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.hppc.ObjectHashSet; import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; +import com.google.common.collect.*; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; @@ -40,10 +37,11 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; @@ -117,6 +115,8 @@ public class MapperService extends AbstractIndexComponent { private volatile ImmutableMap unmappedFieldMappers = ImmutableMap.of(); + private volatile ImmutableSet parentTypes = ImmutableSet.of(); + @Inject public MapperService(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService, IndexFieldDataService fieldDataService, SimilarityLookupService similarityLookupService, @@ -214,7 +214,7 @@ public class MapperService extends AbstractIndexComponent { typeListeners.remove(listener); } - public DocumentMapper merge(String type, CompressedString mappingSource, boolean applyDefault) { + public DocumentMapper merge(String type, CompressedXContent mappingSource, boolean applyDefault) { if (DEFAULT_MAPPING.equals(type)) { // verify we can parse it DocumentMapper mapper = documentParser.parseCompressed(type, mappingSource); @@ -250,6 +250,9 @@ public class MapperService extends AbstractIndexComponent { if (mapper.type().contains(",")) { throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] should not include ',' in it"); } + if (Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0) && mapper.type().equals(mapper.parentFieldMapper().type())) { + throw new IllegalArgumentException("The [_parent.type] option can't point to the same type"); + } if (mapper.type().contains(".") && !PercolatorService.TYPE_NAME.equals(mapper.type())) { logger.warn("Type [{}] contains a '.', it is recommended not to include it within a type name", mapper.type()); } @@ -285,6 +288,12 @@ public class MapperService extends AbstractIndexComponent { typeListener.beforeCreate(mapper); } mappers = newMapBuilder(mappers).put(mapper.type(), mapper).map(); + if (mapper.parentFieldMapper().active()) { + ImmutableSet.Builder parentTypesCopy = ImmutableSet.builder(); + parentTypesCopy.addAll(parentTypes); + parentTypesCopy.add(mapper.parentFieldMapper().type()); + parentTypes = parentTypesCopy.build(); + } assert assertSerialization(mapper); return mapper; } @@ -293,7 +302,7 @@ public class MapperService extends AbstractIndexComponent { private boolean assertSerialization(DocumentMapper mapper) { // capture the source now, it may change due to concurrent parsing - final CompressedString mappingSource = mapper.mappingSource(); + final CompressedXContent mappingSource = mapper.mappingSource(); DocumentMapper newMapper = parse(mapper.type(), mappingSource, false); if (newMapper.mappingSource().equals(mappingSource) == false) { @@ -328,7 +337,7 @@ public class MapperService extends AbstractIndexComponent { this.fieldMappers = this.fieldMappers.copyAndAddAll(fieldMappers); } - public DocumentMapper parse(String mappingType, CompressedString mappingSource, boolean applyDefault) throws MapperParsingException { + public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException { String defaultMappingSource; if (PercolatorService.TYPE_NAME.equals(mappingType)) { defaultMappingSource = this.defaultPercolatorMappingSource; @@ -645,6 +654,10 @@ public class MapperService extends AbstractIndexComponent { return null; } + public ImmutableSet getParentTypes() { + return parentTypes; + } + /** * @return Whether a field is a metadata field. */ diff --git a/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java index 9972ca45cbd..cda0877fdae 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressorFactory; +import org.elasticsearch.common.compress.NotXContentException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContentParser; @@ -144,10 +145,18 @@ public class BinaryFieldMapper extends AbstractFieldMapper { } try { if (indexCreatedBefore2x) { - return CompressorFactory.uncompressIfNeeded(bytes); - } else { - return bytes; + try { + return CompressorFactory.uncompressIfNeeded(bytes); + } catch (NotXContentException e) { + // NOTE: previous versions of Elasticsearch used to try to detect if + // data was compressed. However this could cause decompression failures + // as a user may have submitted arbitrary data which looks like it is + // compressed to elasticsearch but is not. So we removed the ability to + // compress binary fields and keep this empty catch block for backward + // compatibility with 1.x + } } + return bytes; } catch (IOException e) { throw new ElasticsearchParseException("failed to decompress source", e); } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java index 7aca0b17ea6..70109a08074 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.internal; import com.google.common.base.Objects; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.queries.TermsQuery; import org.apache.lucene.search.Query; @@ -33,9 +34,11 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperBuilders; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeResult; @@ -55,7 +58,6 @@ import java.util.Map; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeMapValue; -import static org.elasticsearch.index.mapper.MapperBuilders.parent; /** * @@ -107,7 +109,7 @@ public class ParentFieldMapper extends AbstractFieldMapper implements RootMapper @Override public ParentFieldMapper build(BuilderContext context) { if (type == null) { - throw new MapperParsingException("Parent mapping must contain the parent type"); + throw new MapperParsingException("[_parent] field mapping must contain the [type] option"); } return new ParentFieldMapper(name, indexName, type, fieldDataSettings, context.indexSettings()); } @@ -116,7 +118,7 @@ public class ParentFieldMapper extends AbstractFieldMapper implements RootMapper public static class TypeParser implements Mapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - ParentFieldMapper.Builder builder = parent(); + ParentFieldMapper.Builder builder = MapperBuilders.parent(); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String fieldName = Strings.toUnderscoreCase(entry.getKey()); @@ -145,8 +147,9 @@ public class ParentFieldMapper extends AbstractFieldMapper implements RootMapper private final BytesRef typeAsBytes; protected ParentFieldMapper(String name, String indexName, String type, @Nullable Settings fieldDataSettings, Settings indexSettings) { - super(new Names(name, indexName, indexName, name), Defaults.BOOST, new FieldType(Defaults.FIELD_TYPE), false, - Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, null, null, fieldDataSettings, indexSettings); + super(new Names(name, indexName, indexName, name), Defaults.BOOST, new FieldType(Defaults.FIELD_TYPE), + Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0), Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, + null, null, fieldDataSettings, indexSettings); this.type = type; this.typeAsBytes = type == null ? null : new BytesRef(type); } @@ -181,6 +184,11 @@ public class ParentFieldMapper extends AbstractFieldMapper implements RootMapper @Override protected void parseCreateField(ParseContext context, List fields) throws IOException { + boolean parent = context.docMapper().isParent(context.type()); + if (parent && hasDocValues()) { + fields.add(createJoinField(context.type(), context.id())); + } + if (!active()) { return; } @@ -190,6 +198,9 @@ public class ParentFieldMapper extends AbstractFieldMapper implements RootMapper String parentId = context.parser().text(); context.sourceToParse().parent(parentId); fields.add(new Field(names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType)); + if (hasDocValues()) { + fields.add(createJoinField(type, parentId)); + } } else { // otherwise, we are running it post processing of the xcontent String parsedParentId = context.doc().get(Defaults.NAME); @@ -201,6 +212,9 @@ public class ParentFieldMapper extends AbstractFieldMapper implements RootMapper } // we did not add it in the parsing phase, add it now fields.add(new Field(names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType)); + if (hasDocValues()) { + fields.add(createJoinField(type, parentId)); + } } else if (parentId != null && !parsedParentId.equals(Uid.createUid(context.stringBuilder(), type, parentId))) { throw new MapperParsingException("Parent id mismatch, document value is [" + Uid.createUid(parsedParentId).id() + "], while external value is [" + parentId + "]"); } @@ -209,6 +223,15 @@ public class ParentFieldMapper extends AbstractFieldMapper implements RootMapper // we have parent mapping, yet no value was set, ignore it... } + private SortedDocValuesField createJoinField(String parentType, String id) { + String joinField = joinField(parentType); + return new SortedDocValuesField(joinField, new BytesRef(id)); + } + + public static String joinField(String parentType) { + return ParentFieldMapper.NAME + "#" + parentType; + } + @Override public Uid value(Object value) { if (value == null) { @@ -303,7 +326,9 @@ public class ParentFieldMapper extends AbstractFieldMapper implements RootMapper boolean includeDefaults = params.paramAsBoolean("include_defaults", false); builder.startObject(CONTENT_TYPE); - builder.field("type", type); + if (type != null) { + builder.field("type", type); + } if (customFieldDataSettings != null) { builder.field("fielddata", (Map) customFieldDataSettings.getAsMap()); } else if (includeDefaults) { @@ -316,7 +341,7 @@ public class ParentFieldMapper extends AbstractFieldMapper implements RootMapper @Override public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { ParentFieldMapper other = (ParentFieldMapper) mergeWith; - if (!Objects.equal(type, other.type)) { + if (Objects.equal(type, other.type) == false) { mergeResult.addConflict("The _parent field's type option can't be changed: [" + type + "]->[" + other.type + "]"); } @@ -334,7 +359,7 @@ public class ParentFieldMapper extends AbstractFieldMapper implements RootMapper } /** - * @return Whether the _parent field is actually used. + * @return Whether the _parent field is actually configured. */ public boolean active() { return type != null; diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java index 7c8ae58d5fd..d5d745f263f 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper.internal; import com.google.common.base.Objects; + import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; @@ -31,7 +32,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.compress.CompressedStreamInput; import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -53,7 +53,9 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.RootMapper; import org.elasticsearch.index.mapper.core.AbstractFieldMapper; +import java.io.BufferedInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -324,9 +326,11 @@ public class SourceFieldMapper extends AbstractFieldMapper implements RootMapper // see if we need to convert the content type Compressor compressor = CompressorFactory.compressor(source); if (compressor != null) { - CompressedStreamInput compressedStreamInput = compressor.streamInput(source.streamInput()); + InputStream compressedStreamInput = compressor.streamInput(source.streamInput()); + if (compressedStreamInput.markSupported() == false) { + compressedStreamInput = new BufferedInputStream(compressedStreamInput); + } XContentType contentType = XContentFactory.xContentType(compressedStreamInput); - compressedStreamInput.resetToBufferStart(); if (contentType != formatContentType) { // we need to reread and store back, compressed.... BytesStreamOutput bStream = new BytesStreamOutput(); diff --git a/src/main/java/org/elasticsearch/index/percolator/PercolatorShardModule.java b/src/main/java/org/elasticsearch/index/percolator/PercolatorShardModule.java deleted file mode 100644 index aba7e10fb2e..00000000000 --- a/src/main/java/org/elasticsearch/index/percolator/PercolatorShardModule.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.percolator; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.index.percolator.stats.ShardPercolateService; - -/** - * - */ -public class PercolatorShardModule extends AbstractModule { - - @Override - protected void configure() { - bind(PercolatorQueriesRegistry.class).asEagerSingleton(); - bind(ShardPercolateService.class).asEagerSingleton(); - } -} diff --git a/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java b/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java index d0efd97a695..ecf666ab129 100644 --- a/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java @@ -19,16 +19,23 @@ package org.elasticsearch.index.query; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiDocValues; +import org.apache.lucene.search.*; import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryWrapperFilter; import org.apache.lucene.search.join.BitDocIdSetFilter; import org.elasticsearch.common.ParseField; +import org.apache.lucene.search.join.JoinUtil; +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexParentChildFieldData; import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; @@ -38,6 +45,7 @@ import org.elasticsearch.index.search.child.ChildrenConstantScoreQuery; import org.elasticsearch.index.search.child.ChildrenQuery; import org.elasticsearch.index.search.child.ScoreType; import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; +import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.SubSearchContext; import java.io.IOException; @@ -136,8 +144,9 @@ public class HasChildQueryParser extends BaseQueryParserTemp { if (childDocMapper == null) { throw new QueryParsingException(parseContext, "[has_child] No mapping for for type [" + childType + "]"); } - if (!childDocMapper.parentFieldMapper().active()) { - throw new QueryParsingException(parseContext, "[has_child] Type [" + childType + "] does not have parent mapping"); + ParentFieldMapper parentFieldMapper = childDocMapper.parentFieldMapper(); + if (parentFieldMapper.active() == false) { + throw new QueryParsingException(parseContext, "[has_child] _parent field has no parent type configured"); } if (innerHits != null) { @@ -146,11 +155,6 @@ public class HasChildQueryParser extends BaseQueryParserTemp { parseContext.addInnerHits(name, parentChildInnerHits); } - ParentFieldMapper parentFieldMapper = childDocMapper.parentFieldMapper(); - if (!parentFieldMapper.active()) { - throw new QueryParsingException(parseContext, "[has_child] _parent field not configured"); - } - String parentType = parentFieldMapper.type(); DocumentMapper parentDocMapper = parseContext.mapperService().documentMapper(parentType); if (parentDocMapper == null) { @@ -170,16 +174,20 @@ public class HasChildQueryParser extends BaseQueryParserTemp { // wrap the query with type query innerQuery = Queries.filtered(innerQuery, childDocMapper.typeFilter()); - Query query; - // TODO: use the query API - Filter parentFilter = new QueryWrapperFilter(parentDocMapper.typeFilter()); - ParentChildIndexFieldData parentChildIndexFieldData = parseContext.getForField(parentFieldMapper); - if (minChildren > 1 || maxChildren > 0 || scoreType != ScoreType.NONE) { - query = new ChildrenQuery(parentChildIndexFieldData, parentType, childType, parentFilter, innerQuery, scoreType, minChildren, - maxChildren, shortCircuitParentDocSet, nonNestedDocsFilter); + final Query query; + final ParentChildIndexFieldData parentChildIndexFieldData = parseContext.getForField(parentFieldMapper); + if (parseContext.indexVersionCreated().onOrAfter(Version.V_2_0_0)) { + query = joinUtilHelper(parentType, parentChildIndexFieldData, parentDocMapper.typeFilter(), scoreType, innerQuery, minChildren, maxChildren); } else { - query = new ChildrenConstantScoreQuery(parentChildIndexFieldData, innerQuery, parentType, childType, parentFilter, - shortCircuitParentDocSet, nonNestedDocsFilter); + // TODO: use the query API + Filter parentFilter = new QueryWrapperFilter(parentDocMapper.typeFilter()); + if (minChildren > 1 || maxChildren > 0 || scoreType != ScoreType.NONE) { + query = new ChildrenQuery(parentChildIndexFieldData, parentType, childType, parentFilter, innerQuery, scoreType, minChildren, + maxChildren, shortCircuitParentDocSet, nonNestedDocsFilter); + } else { + query = new ChildrenConstantScoreQuery(parentChildIndexFieldData, innerQuery, parentType, childType, parentFilter, + shortCircuitParentDocSet, nonNestedDocsFilter); + } } if (queryName != null) { parseContext.addNamedQuery(queryName, query); @@ -188,6 +196,46 @@ public class HasChildQueryParser extends BaseQueryParserTemp { return query; } + public static Query joinUtilHelper(String parentType, ParentChildIndexFieldData parentChildIndexFieldData, Query toQuery, ScoreType scoreType, Query innerQuery, int minChildren, int maxChildren) throws IOException { + SearchContext searchContext = SearchContext.current(); + if (searchContext == null) { + throw new IllegalStateException("Search context is required to be set"); + } + + String joinField = ParentFieldMapper.joinField(parentType); + ScoreMode scoreMode; + // TODO: move entirely over from ScoreType to org.apache.lucene.join.ScoreMode, when we drop the 1.x parent child code. + switch (scoreType) { + case NONE: + scoreMode = ScoreMode.None; + break; + case MIN: + scoreMode = ScoreMode.Min; + break; + case MAX: + scoreMode = ScoreMode.Max; + break; + case SUM: + scoreMode = ScoreMode.Total; + break; + case AVG: + scoreMode = ScoreMode.Avg; + break; + default: + throw new UnsupportedOperationException("score type [" + scoreType + "] not supported"); + } + IndexReader indexReader = searchContext.searcher().getIndexReader(); + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + IndexParentChildFieldData indexParentChildFieldData = parentChildIndexFieldData.loadGlobal(indexReader); + MultiDocValues.OrdinalMap ordinalMap = ParentChildIndexFieldData.getOrdinalMap(indexParentChildFieldData, parentType); + + // 0 in pre 2.x p/c impl means unbounded + if (maxChildren == 0) { + maxChildren = Integer.MAX_VALUE; + } + return JoinUtil.createJoinQuery(joinField, innerQuery, toQuery, indexSearcher, scoreMode, ordinalMap, minChildren, maxChildren); + } + @Override public HasChildQueryBuilder getBuilderPrototype() { return HasChildQueryBuilder.PROTOTYPE; diff --git a/src/main/java/org/elasticsearch/index/query/HasParentQueryParser.java b/src/main/java/org/elasticsearch/index/query/HasParentQueryParser.java index 6ff290075d6..e170e94f509 100644 --- a/src/main/java/org/elasticsearch/index/query/HasParentQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/HasParentQueryParser.java @@ -22,6 +22,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; +import org.elasticsearch.Version; import org.apache.lucene.search.QueryWrapperFilter; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; @@ -36,6 +37,7 @@ import org.elasticsearch.index.query.support.InnerHitsQueryParserHelper; import org.elasticsearch.index.query.support.XContentStructure; import org.elasticsearch.index.search.child.ParentConstantScoreQuery; import org.elasticsearch.index.search.child.ParentQuery; +import org.elasticsearch.index.search.child.ScoreType; import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; import org.elasticsearch.search.internal.SubSearchContext; @@ -43,6 +45,8 @@ import java.io.IOException; import java.util.HashSet; import java.util.Set; +import static org.elasticsearch.index.query.HasChildQueryParser.joinUtilHelper; + public class HasParentQueryParser extends BaseQueryParserTemp { private static final ParseField QUERY_FIELD = new ParseField("query", "filter"); @@ -141,7 +145,7 @@ public class HasParentQueryParser extends BaseQueryParserTemp { return query; } - static Query createParentQuery(Query innerQuery, String parentType, boolean score, QueryParseContext parseContext, Tuple innerHits) { + static Query createParentQuery(Query innerQuery, String parentType, boolean score, QueryParseContext parseContext, Tuple innerHits) throws IOException { DocumentMapper parentDocMapper = parseContext.mapperService().documentMapper(parentType); if (parentDocMapper == null) { throw new QueryParsingException(parseContext, "[has_parent] query configured 'parent_type' [" + parentType @@ -196,10 +200,15 @@ public class HasParentQueryParser extends BaseQueryParserTemp { // wrap the query with type query innerQuery = Queries.filtered(innerQuery, parentDocMapper.typeFilter()); Filter childrenFilter = new QueryWrapperFilter(Queries.not(parentFilter)); - if (score) { - return new ParentQuery(parentChildIndexFieldData, innerQuery, parentDocMapper.type(), childrenFilter); + if (parseContext.indexVersionCreated().onOrAfter(Version.V_2_0_0)) { + ScoreType scoreMode = score ? ScoreType.MAX : ScoreType.NONE; + return joinUtilHelper(parentType, parentChildIndexFieldData, childrenFilter, scoreMode, innerQuery, 0, Integer.MAX_VALUE); } else { - return new ParentConstantScoreQuery(parentChildIndexFieldData, innerQuery, parentDocMapper.type(), childrenFilter); + if (score) { + return new ParentQuery(parentChildIndexFieldData, innerQuery, parentDocMapper.type(), childrenFilter); + } else { + return new ParentConstantScoreQuery(parentChildIndexFieldData, innerQuery, parentDocMapper.type(), childrenFilter); + } } } diff --git a/src/main/java/org/elasticsearch/index/query/QueryBuilders.java b/src/main/java/org/elasticsearch/index/query/QueryBuilders.java index 82fc65a5f7e..68ba135cffb 100644 --- a/src/main/java/org/elasticsearch/index/query/QueryBuilders.java +++ b/src/main/java/org/elasticsearch/index/query/QueryBuilders.java @@ -27,7 +27,9 @@ import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.Template; import java.util.Collection; import java.util.Map; @@ -562,6 +564,13 @@ public abstract class QueryBuilders { return new GeoShapeQueryBuilder(name, shape); } + /** + * Facilitates creating template query requests using an inline script + */ + public static TemplateQueryBuilder templateQuery(Template template) { + return new TemplateQueryBuilder(template); + } + /** * Facilitates creating template query requests using an inline script */ @@ -595,6 +604,18 @@ public abstract class QueryBuilders { * * @param script The script to filter by. */ + public static ScriptQueryBuilder scriptQuery(Script script) { + return new ScriptQueryBuilder(script); + } + + /** + * A builder for filter based on a script. + * + * @param script + * The script to filter by. + * @deprecated Use {@link #scriptQuery(Script)} instead. + */ + @Deprecated public static ScriptQueryBuilder scriptQuery(String script) { return new ScriptQueryBuilder(script); } diff --git a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java index 2b87c4cd1a6..b166446ae10 100644 --- a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java +++ b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java @@ -21,6 +21,8 @@ package org.elasticsearch.index.query; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import org.apache.lucene.index.LeafReaderContext; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.queryparser.classic.MapperQueryParser; import org.apache.lucene.queryparser.classic.QueryParserSettings; @@ -79,6 +81,8 @@ public class QueryParseContext { private final Index index; + private final Version indexVersionCreated; + private final IndexQueryParserService indexQueryParser; private final Map namedQueries = Maps.newHashMap(); @@ -99,6 +103,7 @@ public class QueryParseContext { public QueryParseContext(Index index, IndexQueryParserService indexQueryParser) { this.index = index; + this.indexVersionCreated = Version.indexCreated(indexQueryParser.indexSettings()); this.indexQueryParser = indexQueryParser; } @@ -393,4 +398,8 @@ public class QueryParseContext { public boolean isDeprecatedSetting(String setting) { return CACHE.match(setting) || CACHE_KEY.match(setting); } + + public Version indexVersionCreated() { + return indexVersionCreated; + } } diff --git a/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index 41b09eab14b..2ea3383ff87 100644 --- a/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -20,38 +20,60 @@ package org.elasticsearch.index.query; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.Script.ScriptField; import java.io.IOException; +import java.util.HashMap; import java.util.Map; -import static com.google.common.collect.Maps.newHashMap; - public class ScriptQueryBuilder extends QueryBuilder { + private Script script; + public static final String NAME = "script"; - private final String script; + @Deprecated + private String scriptString; + @Deprecated private Map params; + @Deprecated private String lang; private String queryName; - static final ScriptQueryBuilder PROTOTYPE = new ScriptQueryBuilder(null); + static final ScriptQueryBuilder PROTOTYPE = new ScriptQueryBuilder((Script) null); - public ScriptQueryBuilder(String script) { + public ScriptQueryBuilder(Script script) { this.script = script; } + /** + * @deprecated Use {@link #ScriptQueryBuilder(Script)} instead. + */ + @Deprecated + public ScriptQueryBuilder(String script) { + this.scriptString = script; + } + + /** + * @deprecated Use {@link #ScriptQueryBuilder(Script)} instead. + */ + @Deprecated public ScriptQueryBuilder addParam(String name, Object value) { if (params == null) { - params = newHashMap(); + params = new HashMap<>(); } params.put(name, value); return this; } + /** + * @deprecated Use {@link #ScriptQueryBuilder(Script)} instead. + */ + @Deprecated public ScriptQueryBuilder params(Map params) { if (this.params == null) { this.params = params; @@ -63,7 +85,10 @@ public class ScriptQueryBuilder extends QueryBuilder { /** * Sets the script language. + * + * @deprecated Use {@link #ScriptQueryBuilder(Script)} instead. */ + @Deprecated public ScriptQueryBuilder lang(String lang) { this.lang = lang; return this; @@ -78,15 +103,23 @@ public class ScriptQueryBuilder extends QueryBuilder { } @Override - protected void doXContent(XContentBuilder builder, Params params) throws IOException { + protected void doXContent(XContentBuilder builder, Params builderParams) throws IOException { + builder.startObject(NAME); - builder.field("script", script); - if (this.params != null) { - builder.field("params", this.params); - } - if (this.lang != null) { - builder.field("lang", lang); + if (script != null) { + builder.field(ScriptField.SCRIPT.getPreferredName(), script); + } else { + if (this.scriptString != null) { + builder.field("script", scriptString); + } + if (this.params != null) { + builder.field("params", this.params); + } + if (this.lang != null) { + builder.field("lang", lang); + } } + if (queryName != null) { builder.field("_name", queryName); } diff --git a/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java b/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java index 3c275db2b3f..1f5d86544a6 100644 --- a/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.query; +import com.google.common.base.Objects; + import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -29,6 +31,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.script.Script; +import org.elasticsearch.script.Script.ScriptField; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptParameterParser; import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue; @@ -38,7 +41,6 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Map; -import java.util.Objects; import static com.google.common.collect.Maps.newHashMap; @@ -64,13 +66,11 @@ public class ScriptQueryParser extends BaseQueryParserTemp { XContentParser.Token token; // also, when caching, since its isCacheable is false, will result in loading all bit set... - String script = null; - String scriptLang; + Script script = null; Map params = null; String queryName = null; String currentFieldName = null; - ScriptService.ScriptType scriptType = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -78,7 +78,9 @@ public class ScriptQueryParser extends BaseQueryParserTemp { } else if (parseContext.isDeprecatedSetting(currentFieldName)) { // skip } else if (token == XContentParser.Token.START_OBJECT) { - if ("params".equals(currentFieldName)) { + if (ScriptField.SCRIPT.match(currentFieldName)) { + script = Script.parse(parser); + } else if ("params".equals(currentFieldName)) { // TODO remove in 2.0 (here to support old script APIs) params = parser.map(); } else { throw new QueryParsingException(parseContext, "[script] query does not support [" + currentFieldName + "]"); @@ -86,27 +88,29 @@ public class ScriptQueryParser extends BaseQueryParserTemp { } else if (token.isValue()) { if ("_name".equals(currentFieldName)) { queryName = parser.text(); - } else if (!scriptParameterParser.token(currentFieldName, token, parser)){ + } else if (!scriptParameterParser.token(currentFieldName, token, parser)) { throw new QueryParsingException(parseContext, "[script] query does not support [" + currentFieldName + "]"); } } } - ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue(); - if (scriptValue != null) { - script = scriptValue.script(); - scriptType = scriptValue.scriptType(); + if (script == null) { // Didn't find anything using the new API so try using the old one instead + ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue(); + if (scriptValue != null) { + if (params == null) { + params = newHashMap(); + } + script = new Script(scriptValue.script(), scriptValue.scriptType(), scriptParameterParser.lang(), params); + } + } else if (params != null) { + throw new QueryParsingException(parseContext, "script params must be specified inside script object in a [script] filter"); } - scriptLang = scriptParameterParser.lang(); if (script == null) { throw new QueryParsingException(parseContext, "script must be provided with a [script] filter"); } - if (params == null) { - params = newHashMap(); - } - Query query = new ScriptQuery(scriptLang, script, scriptType, params, parseContext.scriptService(), parseContext.lookup()); + Query query = new ScriptQuery(script, parseContext.scriptService(), parseContext.lookup()); if (queryName != null) { parseContext.addNamedQuery(queryName, query); } @@ -115,14 +119,13 @@ public class ScriptQueryParser extends BaseQueryParserTemp { static class ScriptQuery extends Query { - private final String script; - private final Map params; + private final Script script; + private final SearchScript searchScript; - private ScriptQuery(String scriptLang, String script, ScriptService.ScriptType scriptType, Map params, ScriptService scriptService, SearchLookup searchLookup) { + public ScriptQuery(Script script, ScriptService scriptService, SearchLookup searchLookup) { this.script = script; - this.params = params; - this.searchScript = scriptService.search(searchLookup, new Script(scriptLang, script, scriptType, newHashMap(params)), ScriptContext.Standard.SEARCH); + this.searchScript = scriptService.search(searchLookup, script, ScriptContext.Standard.SEARCH); } @Override @@ -135,23 +138,20 @@ public class ScriptQueryParser extends BaseQueryParserTemp { } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (super.equals(o) == false) return false; - - ScriptQuery that = (ScriptQuery) o; - - if (params != null ? !params.equals(that.params) : that.params != null) return false; - if (script != null ? !script.equals(that.script) : that.script != null) return false; - - return true; + public boolean equals(Object obj) { + if (this == obj) + return true; + if (!super.equals(obj)) + return false; + ScriptQuery other = (ScriptQuery) obj; + return Objects.equal(script, other.script); } @Override public int hashCode() { + final int prime = 31; int result = super.hashCode(); - result = 31 * result + Objects.hashCode(script); - result = 31 * result + Objects.hashCode(params); + result = prime * result + Objects.hashCode(script); return result; } diff --git a/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java b/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java index 0ec55907a1e..fe1eb2b0f81 100644 --- a/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java +++ b/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.Template; import java.io.IOException; import java.util.Map; @@ -32,54 +33,61 @@ public class TemplateQueryBuilder extends QueryBuilder { /** Name to reference this type of query. */ public static final String NAME = "template"; + /** Template to fill. */ + private Template template; /** Parameters to fill the template with. */ private Map vars; /** Template to fill.*/ - private String template; + private String templateString; private ScriptService.ScriptType templateType; static final TemplateQueryBuilder PROTOTYPE = new TemplateQueryBuilder(null, null); /** - * @param template the template to use for that query. - * @param vars the parameters to fill the template with. + * @param template + * the template to use for that query. * */ + public TemplateQueryBuilder(Template template) { + this.template = template; + } + + /** + * @param template + * the template to use for that query. + * @param vars + * the parameters to fill the template with. + * @deprecated Use {@link #TemplateQueryBuilder(Template)} instead. + * */ + @Deprecated public TemplateQueryBuilder(String template, Map vars) { this(template, ScriptService.ScriptType.INLINE, vars); } /** - * @param template the template to use for that query. - * @param vars the parameters to fill the template with. - * @param templateType what kind of template (INLINE,FILE,ID) + * @param template + * the template to use for that query. + * @param vars + * the parameters to fill the template with. + * @param templateType + * what kind of template (INLINE,FILE,ID) + * @deprecated Use {@link #TemplateQueryBuilder(Template)} instead. * */ + @Deprecated public TemplateQueryBuilder(String template, ScriptService.ScriptType templateType, Map vars) { - this.template = template; - this.vars =vars; + this.templateString = template; + this.vars = vars; this.templateType = templateType; } @Override - protected void doXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(NAME); - String fieldname; - switch(templateType){ - case FILE: - fieldname = "file"; - break; - case INDEXED: - fieldname = "id"; - break; - case INLINE: - fieldname = TemplateQueryParser.QUERY; - break; - default: - throw new IllegalArgumentException("Unknown template type " + templateType); + protected void doXContent(XContentBuilder builder, Params builderParams) throws IOException { + builder.field(TemplateQueryBuilder.NAME); + if (template == null) { + new Template(templateString, templateType, null, null, this.vars).toXContent(builder, builderParams); + } else { + template.toXContent(builder, builderParams); } - builder.field(fieldname, template); - builder.field(TemplateQueryParser.PARAMS, vars); - builder.endObject(); } @Override diff --git a/src/main/java/org/elasticsearch/index/query/TemplateQueryParser.java b/src/main/java/org/elasticsearch/index/query/TemplateQueryParser.java index d2fbb3140ae..d926e64296f 100644 --- a/src/main/java/org/elasticsearch/index/query/TemplateQueryParser.java +++ b/src/main/java/org/elasticsearch/index/query/TemplateQueryParser.java @@ -22,33 +22,29 @@ import org.apache.lucene.search.Query; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.mustache.MustacheScriptEngineService; +import org.elasticsearch.script.Template; import java.io.IOException; import java.util.HashMap; import java.util.Map; /** - * In the simplest case, parse template string and variables from the request, compile the template and - * execute the template against the given variables. + * In the simplest case, parse template string and variables from the request, + * compile the template and execute the template against the given variables. * */ public class TemplateQueryParser extends BaseQueryParserTemp { /** Name of query parameter containing the template string. */ public static final String QUERY = "query"; - /** Name of query parameter containing the template parameters. */ - public static final String PARAMS = "params"; private final ScriptService scriptService; - private final static Map parametersToTypes = new HashMap<>(); + private final static Map parametersToTypes = new HashMap<>(); static { parametersToTypes.put("query", ScriptService.ScriptType.INLINE); parametersToTypes.put("file", ScriptService.ScriptType.FILE); @@ -66,17 +62,19 @@ public class TemplateQueryParser extends BaseQueryParserTemp { } /** - * Parses the template query replacing template parameters with provided values. - * Handles both submitting the template as part of the request as well as - * referencing only the template name. - * @param parseContext parse context containing the templated query. + * Parses the template query replacing template parameters with provided + * values. Handles both submitting the template as part of the request as + * well as referencing only the template name. + * + * @param parseContext + * parse context containing the templated query. */ @Override @Nullable public Query parse(QueryParseContext parseContext) throws IOException { XContentParser parser = parseContext.parser(); - TemplateContext templateContext = parse(parser, PARAMS, parametersToTypes); - ExecutableScript executable = this.scriptService.executable(new Script(MustacheScriptEngineService.NAME, templateContext.template(), templateContext.scriptType(), templateContext.params()), ScriptContext.Standard.SEARCH); + Template template = parse(parser); + ExecutableScript executable = this.scriptService.executable(template, ScriptContext.Standard.SEARCH); BytesReference querySource = (BytesReference) executable.run(); @@ -87,73 +85,30 @@ public class TemplateQueryParser extends BaseQueryParserTemp { } } - public static TemplateContext parse(XContentParser parser, String paramsFieldname, String ... parameters) throws IOException { + public static Template parse(XContentParser parser, String... parameters) throws IOException { - Map parameterMap = new HashMap<>(parametersToTypes); + Map parameterMap = new HashMap<>(parametersToTypes); for (String parameter : parameters) { parameterMap.put(parameter, ScriptService.ScriptType.INLINE); } - return parse(parser,paramsFieldname,parameterMap); + return parse(parser, parameterMap); } - public static TemplateContext parse(XContentParser parser, String paramsFieldname) throws IOException { - return parse(parser,paramsFieldname,parametersToTypes); + public static Template parse(String defaultLang, XContentParser parser, String... parameters) throws IOException { + + Map parameterMap = new HashMap<>(parametersToTypes); + for (String parameter : parameters) { + parameterMap.put(parameter, ScriptService.ScriptType.INLINE); + } + return Template.parse(parser, parameterMap, defaultLang); } - public static TemplateContext parse(XContentParser parser, String paramsFieldname, Map parameterMap) throws IOException { - Map params = null; - String templateNameOrTemplateContent = null; - - String currentFieldName = null; - XContentParser.Token token; - ScriptService.ScriptType type = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (parameterMap.containsKey(currentFieldName)) { - type = parameterMap.get(currentFieldName); - if (token == XContentParser.Token.START_OBJECT) { - XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent()); - builder.copyCurrentStructure(parser); - templateNameOrTemplateContent = builder.string(); - } else { - templateNameOrTemplateContent = parser.text(); - } - } else if (paramsFieldname.equals(currentFieldName)) { - params = parser.map(); - } - } - - return new TemplateContext(type, templateNameOrTemplateContent, params); + public static Template parse(XContentParser parser) throws IOException { + return parse(parser, parametersToTypes); } - public static class TemplateContext { - private Map params; - private String template; - private ScriptService.ScriptType type; - - public TemplateContext(ScriptService.ScriptType type, String template, Map params) { - this.params = params; - this.template = template; - this.type = type; - } - - public Map params() { - return params; - } - - public String template() { - return template; - } - - public ScriptService.ScriptType scriptType(){ - return type; - } - - @Override - public String toString(){ - return type + " " + template; - } + public static Template parse(XContentParser parser, Map parameterMap) throws IOException { + return Template.parse(parser, parameterMap); } @Override diff --git a/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilders.java b/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilders.java index 8ae38b5008f..ef9865395b3 100644 --- a/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilders.java +++ b/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionBuilders.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.query.functionscore.lin.LinearDecayFunctionBuilde import org.elasticsearch.index.query.functionscore.random.RandomScoreFunctionBuilder; import org.elasticsearch.index.query.functionscore.script.ScriptScoreFunctionBuilder; import org.elasticsearch.index.query.functionscore.weight.WeightBuilder; +import org.elasticsearch.script.Script; import java.util.Map; @@ -56,18 +57,38 @@ public class ScoreFunctionBuilders { return new LinearDecayFunctionBuilder(fieldName, null, scale); } + public static ScriptScoreFunctionBuilder scriptFunction(Script script) { + return (new ScriptScoreFunctionBuilder()).script(script); + } + + /** + * @deprecated Use {@link #scriptFunction(Script)} instead. + */ + @Deprecated public static ScriptScoreFunctionBuilder scriptFunction(String script) { return (new ScriptScoreFunctionBuilder()).script(script); } + /** + * @deprecated Use {@link #scriptFunction(Script)} instead. + */ + @Deprecated public static ScriptScoreFunctionBuilder scriptFunction(String script, String lang) { return (new ScriptScoreFunctionBuilder()).script(script).lang(lang); } + /** + * @deprecated Use {@link #scriptFunction(Script)} instead. + */ + @Deprecated public static ScriptScoreFunctionBuilder scriptFunction(String script, String lang, Map params) { return (new ScriptScoreFunctionBuilder()).script(script).lang(lang).params(params); } + /** + * @deprecated Use {@link #scriptFunction(Script)} instead. + */ + @Deprecated public static ScriptScoreFunctionBuilder scriptFunction(String script, Map params) { return (new ScriptScoreFunctionBuilder()).script(script).params(params); } diff --git a/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionBuilder.java b/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionBuilder.java index 3f715512bab..20dca88788a 100644 --- a/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionBuilder.java +++ b/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionBuilder.java @@ -19,12 +19,13 @@ package org.elasticsearch.index.query.functionscore.script; -import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; - -import com.google.common.collect.Maps; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.Script.ScriptField; import java.io.IOException; +import java.util.HashMap; import java.util.Map; /** @@ -33,7 +34,9 @@ import java.util.Map; */ public class ScriptScoreFunctionBuilder extends ScoreFunctionBuilder { - private String script; + private Script script; + + private String scriptString; private String lang; @@ -43,22 +46,35 @@ public class ScriptScoreFunctionBuilder extends ScoreFunctionBuilder { } - public ScriptScoreFunctionBuilder script(String script) { + public ScriptScoreFunctionBuilder script(Script script) { this.script = script; return this; } /** - * Sets the language of the script. + * @deprecated Use {@link #script(Script)} instead */ + @Deprecated + public ScriptScoreFunctionBuilder script(String script) { + this.scriptString = script; + return this; + } + + /** + * Sets the language of the script.@deprecated Use {@link #script(Script)} + * instead + */ + @Deprecated public ScriptScoreFunctionBuilder lang(String lang) { this.lang = lang; return this; } /** - * Additional parameters that can be provided to the script. + * Additional parameters that can be provided to the script.@deprecated Use + * {@link #script(Script)} instead */ + @Deprecated public ScriptScoreFunctionBuilder params(Map params) { if (this.params == null) { this.params = params; @@ -69,11 +85,13 @@ public class ScriptScoreFunctionBuilder extends ScoreFunctionBuilder { } /** - * Additional parameters that can be provided to the script. + * Additional parameters that can be provided to the script.@deprecated Use + * {@link #script(Script)} instead */ + @Deprecated public ScriptScoreFunctionBuilder param(String key, Object value) { if (params == null) { - params = Maps.newHashMap(); + params = new HashMap<>(); } params.put(key, value); return this; @@ -82,12 +100,18 @@ public class ScriptScoreFunctionBuilder extends ScoreFunctionBuilder { @Override public void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(getName()); - builder.field("script", script); - if (lang != null) { - builder.field("lang", lang); - } - if (this.params != null) { - builder.field("params", this.params); + if (script != null) { + builder.field(ScriptField.SCRIPT.getPreferredName(), script); + } else { + if (scriptString != null) { + builder.field("script", scriptString); + } + if (lang != null) { + builder.field("lang", lang); + } + if (this.params != null) { + builder.field("params", this.params); + } } builder.endObject(); } diff --git a/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionParser.java b/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionParser.java index b01eaee3615..72a592da5b3 100644 --- a/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionParser.java +++ b/src/main/java/org/elasticsearch/index/query/functionscore/script/ScriptScoreFunctionParser.java @@ -29,15 +29,17 @@ import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryParsingException; import org.elasticsearch.index.query.functionscore.ScoreFunctionParser; import org.elasticsearch.script.Script; +import org.elasticsearch.script.Script.ScriptField; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptParameterParser; import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.SearchScript; import java.io.IOException; import java.util.Map; +import static com.google.common.collect.Maps.newHashMap; + /** * */ @@ -57,16 +59,17 @@ public class ScriptScoreFunctionParser implements ScoreFunctionParser { @Override public ScoreFunction parse(QueryParseContext parseContext, XContentParser parser) throws IOException, QueryParsingException { ScriptParameterParser scriptParameterParser = new ScriptParameterParser(); - String script = null; + Script script = null; Map vars = null; - ScriptService.ScriptType scriptType = null; String currentFieldName = null; XContentParser.Token token; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("params".equals(currentFieldName)) { + if (ScriptField.SCRIPT.match(currentFieldName)) { + script = Script.parse(parser); + } else if ("params".equals(currentFieldName)) { // TODO remove in 2.0 (here to support old script APIs) vars = parser.map(); } else { throw new QueryParsingException(parseContext, NAMES[0] + " query does not support [" + currentFieldName + "]"); @@ -78,19 +81,26 @@ public class ScriptScoreFunctionParser implements ScoreFunctionParser { } } - ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue(); - if (scriptValue != null) { - script = scriptValue.script(); - scriptType = scriptValue.scriptType(); + if (script == null) { // Didn't find anything using the new API so try using the old one instead + ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue(); + if (scriptValue != null) { + if (vars == null) { + vars = newHashMap(); + } + script = new Script(scriptValue.script(), scriptValue.scriptType(), scriptParameterParser.lang(), vars); + } + } else if (vars != null) { + throw new QueryParsingException(parseContext, "script params must be specified inside script object"); } + if (script == null) { throw new QueryParsingException(parseContext, NAMES[0] + " requires 'script' field"); } SearchScript searchScript; try { - searchScript = parseContext.scriptService().search(parseContext.lookup(), new Script(scriptParameterParser.lang(), script, scriptType, vars), ScriptContext.Standard.SEARCH); - return new ScriptScoreFunction(script, vars, searchScript); + searchScript = parseContext.scriptService().search(parseContext.lookup(), script, ScriptContext.Standard.SEARCH); + return new ScriptScoreFunction(script, searchScript); } catch (Exception e) { throw new QueryParsingException(parseContext, NAMES[0] + " the script could not be loaded", e); } diff --git a/src/main/java/org/elasticsearch/index/query/support/BaseInnerHitBuilder.java b/src/main/java/org/elasticsearch/index/query/support/BaseInnerHitBuilder.java index 0987e5dd4f6..8e991c8f130 100644 --- a/src/main/java/org/elasticsearch/index/query/support/BaseInnerHitBuilder.java +++ b/src/main/java/org/elasticsearch/index/query/support/BaseInnerHitBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.script.Script; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.sort.SortBuilder; @@ -148,33 +149,60 @@ public abstract class BaseInnerHitBuilder impleme * @param name The name that will represent this value in the return hit * @param script The script to use */ + public T addScriptField(String name, Script script) { + sourceBuilder().scriptField(name, script); + return (T) this; + } + + /** + * Adds a script based field to load and return. The field does not have to + * be stored, but its recommended to use non analyzed or numeric fields. + * + * @param name + * The name that will represent this value in the return hit + * @param script + * The script to use + * @deprecated Use {@link #addScriptField(String, Script)} instead. + */ + @Deprecated public T addScriptField(String name, String script) { sourceBuilder().scriptField(name, script); return (T) this; } /** - * Adds a script based field to load and return. The field does not have to be stored, - * but its recommended to use non analyzed or numeric fields. + * Adds a script based field to load and return. The field does not have to + * be stored, but its recommended to use non analyzed or numeric fields. * - * @param name The name that will represent this value in the return hit - * @param script The script to use - * @param params Parameters that the script can use. + * @param name + * The name that will represent this value in the return hit + * @param script + * The script to use + * @param params + * Parameters that the script can use. + * @deprecated Use {@link #addScriptField(String, Script)} instead. */ + @Deprecated public T addScriptField(String name, String script, Map params) { sourceBuilder().scriptField(name, script, params); return (T) this; } /** - * Adds a script based field to load and return. The field does not have to be stored, - * but its recommended to use non analyzed or numeric fields. + * Adds a script based field to load and return. The field does not have to + * be stored, but its recommended to use non analyzed or numeric fields. * - * @param name The name that will represent this value in the return hit - * @param lang The language of the script - * @param script The script to use - * @param params Parameters that the script can use (can be null). + * @param name + * The name that will represent this value in the return hit + * @param lang + * The language of the script + * @param script + * The script to use + * @param params + * Parameters that the script can use (can be null). + * @deprecated Use {@link #addScriptField(String, Script)} instead. */ + @Deprecated public T addScriptField(String name, String lang, String script, Map params) { sourceBuilder().scriptField(name, lang, script, params); return (T) this; diff --git a/src/main/java/org/elasticsearch/index/search/stats/ShardSearchModule.java b/src/main/java/org/elasticsearch/index/search/stats/ShardSearchModule.java deleted file mode 100644 index 28f8c09c8c2..00000000000 --- a/src/main/java/org/elasticsearch/index/search/stats/ShardSearchModule.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.search.stats; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.index.search.slowlog.ShardSlowLogSearchService; - -/** - */ -public class ShardSearchModule extends AbstractModule { - - @Override - protected void configure() { - bind(ShardSearchService.class).asEagerSingleton(); - bind(ShardSlowLogSearchService.class).asEagerSingleton(); - } -} diff --git a/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java b/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java index fc44f11eab9..672b63bfb1c 100644 --- a/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java +++ b/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java @@ -22,8 +22,27 @@ package org.elasticsearch.index.shard; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.ShardLock; +import org.elasticsearch.index.cache.bitset.ShardBitsetFilterCache; +import org.elasticsearch.index.cache.filter.ShardFilterCache; +import org.elasticsearch.index.cache.query.ShardQueryCache; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.fielddata.ShardFieldData; +import org.elasticsearch.index.gateway.IndexShardGateway; +import org.elasticsearch.index.gateway.IndexShardGatewayService; +import org.elasticsearch.index.get.ShardGetService; +import org.elasticsearch.index.indexing.ShardIndexingService; +import org.elasticsearch.index.indexing.slowlog.ShardSlowLogIndexingService; +import org.elasticsearch.index.percolator.PercolatorQueriesRegistry; +import org.elasticsearch.index.percolator.stats.ShardPercolateService; +import org.elasticsearch.index.search.slowlog.ShardSlowLogSearchService; +import org.elasticsearch.index.search.stats.ShardSearchService; +import org.elasticsearch.index.snapshots.IndexShardSnapshotAndRestoreService; +import org.elasticsearch.index.store.DirectoryService; +import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.suggest.stats.ShardSuggestService; +import org.elasticsearch.index.termvectors.ShardTermVectorsService; import org.elasticsearch.index.translog.TranslogService; import org.elasticsearch.index.warmer.ShardIndexWarmerService; @@ -43,9 +62,11 @@ public class IndexShardModule extends AbstractModule { private final ShardId shardId; private final Settings settings; private final boolean primary; + private final ShardFilterCache shardFilterCache; - public IndexShardModule(ShardId shardId, boolean primary, Settings settings) { + public IndexShardModule(ShardId shardId, boolean primary, Settings settings, ShardFilterCache shardFilterCache) { this.settings = settings; + this.shardFilterCache = shardFilterCache; this.shardId = shardId; this.primary = primary; if (settings.get("index.translog.type") != null) { @@ -69,7 +90,25 @@ public class IndexShardModule extends AbstractModule { } bind(EngineFactory.class).to(settings.getAsClass(ENGINE_FACTORY, DEFAULT_ENGINE_FACTORY_CLASS, ENGINE_PREFIX, ENGINE_SUFFIX)); + bind(ShardIndexWarmerService.class).asEagerSingleton(); + bind(ShardIndexingService.class).asEagerSingleton(); + bind(ShardSlowLogIndexingService.class).asEagerSingleton(); + bind(ShardSearchService.class).asEagerSingleton(); + bind(ShardSlowLogSearchService.class).asEagerSingleton(); + bind(ShardGetService.class).asEagerSingleton(); + bind(ShardFilterCache.class).toInstance(shardFilterCache); + bind(ShardQueryCache.class).asEagerSingleton(); + bind(ShardBitsetFilterCache.class).asEagerSingleton(); + bind(ShardFieldData.class).asEagerSingleton(); + bind(IndexShardGateway.class).asEagerSingleton(); + bind(IndexShardGatewayService.class).asEagerSingleton(); + bind(PercolatorQueriesRegistry.class).asEagerSingleton(); + bind(ShardPercolateService.class).asEagerSingleton(); + bind(ShardTermVectorsService.class).asEagerSingleton(); + bind(IndexShardSnapshotAndRestoreService.class).asEagerSingleton(); + bind(ShardSuggestService.class).asEagerSingleton(); } + } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotModule.java b/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotModule.java deleted file mode 100644 index c0cf9788400..00000000000 --- a/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotModule.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.snapshots; - -import org.elasticsearch.common.inject.AbstractModule; - -/** - * This shard-level module configures {@link IndexShardSnapshotAndRestoreService} - */ -public class IndexShardSnapshotModule extends AbstractModule { - - @Override - protected void configure() { - bind(IndexShardSnapshotAndRestoreService.class).asEagerSingleton(); - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java index 707e77dbefe..d498119c5fa 100644 --- a/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java +++ b/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java @@ -19,9 +19,11 @@ package org.elasticsearch.index.snapshots.blobstore; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.io.ByteStreams; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; @@ -41,8 +43,11 @@ import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.store.InputStreamIndexInput; import org.elasticsearch.common.settings.Settings; @@ -65,10 +70,10 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.*; -import java.util.concurrent.CopyOnWriteArrayList; import static com.google.common.collect.Lists.newArrayList; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.testBlobPrefix; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.toStreamOutput; /** * Blob store based implementation of IndexShardRepository @@ -96,7 +101,15 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements private RateLimitingInputStream.Listener snapshotThrottleListener; - private static final String SNAPSHOT_PREFIX = "snapshot-"; + private boolean compress; + + protected static final String SNAPSHOT_PREFIX = "snapshot-"; + + protected static final String SNAPSHOT_INDEX_PREFIX = "index-"; + + protected static final String SNAPSHOT_TEMP_PREFIX = "pending-"; + + protected static final String DATA_BLOB_PREFIX = "__"; @Inject public BlobStoreIndexShardRepository(Settings settings, RepositoryName repositoryName, IndicesService indicesService, ClusterService clusterService) { @@ -115,7 +128,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements */ public void initialize(BlobStore blobStore, BlobPath basePath, ByteSizeValue chunkSize, RateLimiter snapshotRateLimiter, RateLimiter restoreRateLimiter, - final RateLimiterListener rateLimiterListener) { + final RateLimiterListener rateLimiterListener, boolean compress) { this.blobStore = blobStore; this.basePath = basePath; this.chunkSize = chunkSize; @@ -128,6 +141,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements rateLimiterListener.onSnapshotPause(nanos); } }; + this.compress = compress; } /** @@ -232,11 +246,11 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements * Serializes snapshot to JSON * * @param snapshot snapshot - * @param stream the stream to output the snapshot JSON represetation to + * @param output the stream to output the snapshot JSON representation to * @throws IOException if an IOException occurs */ - public static void writeSnapshot(BlobStoreIndexShardSnapshot snapshot, OutputStream stream) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream).prettyPrint(); + public void writeSnapshot(BlobStoreIndexShardSnapshot snapshot, StreamOutput output) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, output).prettyPrint(); BlobStoreIndexShardSnapshot.toXContent(snapshot, builder, ToXContent.EMPTY_PARAMS); builder.flush(); } @@ -249,12 +263,36 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements * @throws IOException if an IOException occurs */ public static BlobStoreIndexShardSnapshot readSnapshot(InputStream stream) throws IOException { - try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(stream)) { + byte[] data = ByteStreams.toByteArray(stream); + try (XContentParser parser = XContentHelper.createParser(new BytesArray(data))) { parser.nextToken(); return BlobStoreIndexShardSnapshot.fromXContent(parser); } } + /** + * Parses JSON representation of a snapshot + * + * @param stream JSON + * @return snapshot + * @throws IOException if an IOException occurs + * */ + public static BlobStoreIndexShardSnapshots readSnapshots(InputStream stream) throws IOException { + byte[] data = ByteStreams.toByteArray(stream); + try (XContentParser parser = XContentHelper.createParser(new BytesArray(data))) { + parser.nextToken(); + return BlobStoreIndexShardSnapshots.fromXContent(parser); + } + } + /** + * Returns true if metadata files should be compressed + * + * @return true if compression is needed + */ + protected boolean isCompress() { + return compress; + } + /** * Context for snapshot/restore operations */ @@ -287,7 +325,9 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements throw new IndexShardSnapshotException(shardId, "Failed to list content of gateway", e); } - BlobStoreIndexShardSnapshots snapshots = buildBlobStoreIndexShardSnapshots(blobs); + Tuple tuple = buildBlobStoreIndexShardSnapshots(blobs); + BlobStoreIndexShardSnapshots snapshots = tuple.v1(); + int fileListGeneration = tuple.v2(); String commitPointName = snapshotBlobName(snapshotId); @@ -297,15 +337,15 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements logger.debug("[{}] [{}] failed to delete shard snapshot file", shardId, snapshotId); } - // delete all files that are not referenced by any commit point - // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones - List newSnapshotsList = Lists.newArrayList(); - for (BlobStoreIndexShardSnapshot point : snapshots) { + // Build a list of snapshots that should be preserved + List newSnapshotsList = Lists.newArrayList(); + for (SnapshotFiles point : snapshots) { if (!point.snapshot().equals(snapshotId.getSnapshot())) { newSnapshotsList.add(point); } } - cleanup(newSnapshotsList, blobs); + // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index + finalize(newSnapshotsList, fileListGeneration + 1, blobs); } /** @@ -322,26 +362,63 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements } /** - * Removes all unreferenced files from the repository + * Removes all unreferenced files from the repository and writes new index file + * + * We need to be really careful in handling index files in case of failures to make sure we have index file that + * points to files that were deleted. + * * * @param snapshots list of active snapshots in the container + * @param fileListGeneration the generation number of the snapshot index file * @param blobs list of blobs in the container */ - protected void cleanup(List snapshots, ImmutableMap blobs) { + protected void finalize(List snapshots, int fileListGeneration, ImmutableMap blobs) { BlobStoreIndexShardSnapshots newSnapshots = new BlobStoreIndexShardSnapshots(snapshots); - // now go over all the blobs, and if they don't exists in a snapshot, delete them + // delete old index files first for (String blobName : blobs.keySet()) { - if (!blobName.startsWith("__")) { - continue; - } - if (newSnapshots.findNameFile(FileInfo.canonicalName(blobName)) == null) { + // delete old file lists + if (blobName.startsWith(SNAPSHOT_TEMP_PREFIX) || blobName.startsWith(SNAPSHOT_INDEX_PREFIX)) { try { blobContainer.deleteBlob(blobName); } catch (IOException e) { - logger.debug("[{}] [{}] error deleting blob [{}] during cleanup", e, snapshotId, shardId, blobName); + // We cannot delete index file - this is fatal, we cannot continue, otherwise we might end up + // with references to non-existing files + throw new IndexShardSnapshotFailedException(shardId, "error deleting index file [{}] during cleanup", e); } } } + + // now go over all the blobs, and if they don't exists in a snapshot, delete them + for (String blobName : blobs.keySet()) { + // delete old file lists + if (blobName.startsWith(DATA_BLOB_PREFIX)) { + if (newSnapshots.findNameFile(FileInfo.canonicalName(blobName)) == null) { + try { + blobContainer.deleteBlob(blobName); + } catch (IOException e) { + logger.debug("[{}] [{}] error deleting blob [{}] during cleanup", e, snapshotId, shardId, blobName); + } + } + } + } + + // If we deleted all snapshots - we don't need to create the index file + if (snapshots.size() > 0) { + String newSnapshotIndexName = SNAPSHOT_INDEX_PREFIX + fileListGeneration; + try (OutputStream output = blobContainer.createOutput(SNAPSHOT_TEMP_PREFIX + fileListGeneration)) { + StreamOutput stream = compressIfNeeded(output); + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream); + newSnapshots.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.flush(); + } catch (IOException e) { + throw new IndexShardSnapshotFailedException(shardId, "Failed to write file list", e); + } + try { + blobContainer.move(SNAPSHOT_TEMP_PREFIX + fileListGeneration, newSnapshotIndexName); + } catch (IOException e) { + throw new IndexShardSnapshotFailedException(shardId, "Failed to rename file list", e); + } + } } /** @@ -351,7 +428,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements * @return the blob name */ protected String fileNameFromGeneration(long generation) { - return "__" + Long.toString(generation, Character.MAX_RADIX); + return DATA_BLOB_PREFIX + Long.toString(generation, Character.MAX_RADIX); } /** @@ -363,17 +440,17 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements protected long findLatestFileNameGeneration(ImmutableMap blobs) { long generation = -1; for (String name : blobs.keySet()) { - if (!name.startsWith("__")) { + if (!name.startsWith(DATA_BLOB_PREFIX)) { continue; } name = FileInfo.canonicalName(name); try { - long currentGen = Long.parseLong(name.substring(2) /*__*/, Character.MAX_RADIX); + long currentGen = Long.parseLong(name.substring(DATA_BLOB_PREFIX.length()), Character.MAX_RADIX); if (currentGen > generation) { generation = currentGen; } } catch (NumberFormatException e) { - logger.warn("file [{}] does not conform to the '__' schema"); + logger.warn("file [{}] does not conform to the '{}' schema", name, DATA_BLOB_PREFIX); } } return generation; @@ -383,20 +460,47 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements * Loads all available snapshots in the repository * * @param blobs list of blobs in repository - * @return BlobStoreIndexShardSnapshots + * @return tuple of BlobStoreIndexShardSnapshots and the last snapshot index generation */ - protected BlobStoreIndexShardSnapshots buildBlobStoreIndexShardSnapshots(ImmutableMap blobs) { - List snapshots = Lists.newArrayList(); + protected Tuple buildBlobStoreIndexShardSnapshots(ImmutableMap blobs) { + int latest = -1; + for (String name : blobs.keySet()) { + if (name.startsWith(SNAPSHOT_INDEX_PREFIX)) { + try { + int gen = Integer.parseInt(name.substring(SNAPSHOT_INDEX_PREFIX.length())); + if (gen > latest) { + latest = gen; + } + } catch (NumberFormatException ex) { + logger.warn("failed to parse index file name [{}]", name); + } + } + } + if (latest >= 0) { + try (InputStream stream = blobContainer.openInput(SNAPSHOT_INDEX_PREFIX + latest)) { + return new Tuple<>(readSnapshots(stream), latest); + } catch (IOException e) { + logger.warn("failed to read index file [{}]", e, SNAPSHOT_INDEX_PREFIX + latest); + } + } + + // We couldn't load the index file - falling back to loading individual snapshots + List snapshots = Lists.newArrayList(); for (String name : blobs.keySet()) { if (name.startsWith(SNAPSHOT_PREFIX)) { try (InputStream stream = blobContainer.openInput(name)) { - snapshots.add(readSnapshot(stream)); + BlobStoreIndexShardSnapshot snapshot = readSnapshot(stream); + snapshots.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles())); } catch (IOException e) { logger.warn("failed to read commit point [{}]", e, name); } } } - return new BlobStoreIndexShardSnapshots(snapshots); + return new Tuple<>(new BlobStoreIndexShardSnapshots(snapshots), -1); + } + + protected StreamOutput compressIfNeeded(OutputStream output) throws IOException { + return toStreamOutput(output, isCompress()); } } @@ -442,9 +546,10 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements } long generation = findLatestFileNameGeneration(blobs); - BlobStoreIndexShardSnapshots snapshots = buildBlobStoreIndexShardSnapshots(blobs); + Tuple tuple = buildBlobStoreIndexShardSnapshots(blobs); + BlobStoreIndexShardSnapshots snapshots = tuple.v1(); + int fileListGeneration = tuple.v2(); - final CopyOnWriteArrayList failures = new CopyOnWriteArrayList<>(); final List indexCommitPointFiles = newArrayList(); int indexNumberOfFiles = 0; @@ -464,23 +569,28 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements } logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName); final StoreFileMetaData md = metadata.get(fileName); - boolean snapshotRequired = false; - BlobStoreIndexShardSnapshot.FileInfo fileInfo = snapshots.findPhysicalIndexFile(fileName); - try { - // in 1.3.3 we added additional hashes for .si / segments_N files - // to ensure we don't double the space in the repo since old snapshots - // don't have this hash we try to read that hash from the blob store - // in a bwc compatible way. - maybeRecalculateMetadataHash(blobContainer, fileInfo, metadata); - } catch (Throwable e) { - logger.warn("{} Can't calculate hash from blob for file [{}] [{}]", e, shardId, fileInfo.physicalName(), fileInfo.metadata()); + FileInfo existingFileInfo = null; + ImmutableList filesInfo = snapshots.findPhysicalIndexFiles(fileName); + if (filesInfo != null) { + for (FileInfo fileInfo : filesInfo) { + try { + // in 1.3.3 we added additional hashes for .si / segments_N files + // to ensure we don't double the space in the repo since old snapshots + // don't have this hash we try to read that hash from the blob store + // in a bwc compatible way. + maybeRecalculateMetadataHash(blobContainer, fileInfo, metadata); + } catch (Throwable e) { + logger.warn("{} Can't calculate hash from blob for file [{}] [{}]", e, shardId, fileInfo.physicalName(), fileInfo.metadata()); + } + if (fileInfo.isSame(md) && snapshotFileExistsInBlobs(fileInfo, blobs)) { + // a commit point file with the same name, size and checksum was already copied to repository + // we will reuse it for this snapshot + existingFileInfo = fileInfo; + break; + } + } } - if (fileInfo == null || !fileInfo.isSame(md) || !snapshotFileExistsInBlobs(fileInfo, blobs)) { - // commit point file does not exists in any commit point, or has different length, or does not fully exists in the listed blobs - snapshotRequired = true; - } - - if (snapshotRequired) { + if (existingFileInfo == null) { indexNumberOfFiles++; indexTotalFilesSize += md.length(); // create a new FileInfo @@ -488,7 +598,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements indexCommitPointFiles.add(snapshotFileInfo); filesToSnapshot.add(snapshotFileInfo); } else { - indexCommitPointFiles.add(fileInfo); + indexCommitPointFiles.add(existingFileInfo); } } @@ -515,7 +625,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements System.currentTimeMillis() - snapshotStatus.startTime(), indexNumberOfFiles, indexTotalFilesSize); //TODO: The time stored in snapshot doesn't include cleanup time. logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId); - try (OutputStream output = blobContainer.createOutput(snapshotBlobName)) { + try (StreamOutput output = compressIfNeeded(blobContainer.createOutput(snapshotBlobName))) { writeSnapshot(snapshot, output); } catch (IOException e) { throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e); @@ -523,12 +633,13 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements // delete all files that are not referenced by any commit point // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones - List newSnapshotsList = Lists.newArrayList(); - newSnapshotsList.add(snapshot); - for (BlobStoreIndexShardSnapshot point : snapshots) { + List newSnapshotsList = Lists.newArrayList(); + newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles())); + for (SnapshotFiles point : snapshots) { newSnapshotsList.add(point); } - cleanup(newSnapshotsList, blobs); + // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index + finalize(newSnapshotsList, fileListGeneration + 1, blobs); snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.DONE); } finally { store.decRef(); @@ -709,6 +820,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements try { logger.debug("[{}] [{}] restoring to [{}] ...", snapshotId, repositoryName, shardId); BlobStoreIndexShardSnapshot snapshot = loadSnapshot(); + SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()); final Store.MetadataSnapshot recoveryTargetMetadata; try { recoveryTargetMetadata = store.getMetadataOrEmpty(); @@ -786,6 +898,22 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements throw new IndexShardRestoreFailedException(shardId, "Failed to fetch index version after copying it over", e); } recoveryState.getIndex().updateVersion(segmentCommitInfos.getVersion()); + + /// now, go over and clean files that are in the store, but were not in the snapshot + try { + for (String storeFile : store.directory().listAll()) { + if (!Store.isChecksum(storeFile) && !snapshotFiles.containPhysicalIndexFile(storeFile)) { + try { + store.deleteQuiet("restore", storeFile); + store.directory().deleteFile(storeFile); + } catch (IOException e) { + logger.warn("[{}] failed to delete file [{}] during snapshot cleanup", snapshotId, storeFile); + } + } + } + } catch (IOException e) { + logger.warn("[{}] failed to list directory - some of files might not be deleted", snapshotId); + } } finally { store.decRef(); } diff --git a/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index b643bf234e5..e2acff96f57 100644 --- a/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -190,6 +190,24 @@ public class BlobStoreIndexShardSnapshot { return metadata.isSame(md); } + /** + * Checks if a file in a store is the same file + * + * @param fileInfo file in a store + * @return true if file in a store this this file have the same checksum and length + */ + public boolean isSame(FileInfo fileInfo) { + if (numberOfParts != fileInfo.numberOfParts) return false; + if (partBytes != fileInfo.partBytes) return false; + if (!name.equals(fileInfo.name)) return false; + if (partSize != null) { + if (!partSize.equals(fileInfo.partSize)) return false; + } else { + if (fileInfo.partSize != null) return false; + } + return metadata.isSame(fileInfo.metadata); + } + static final class Fields { static final XContentBuilderString NAME = new XContentBuilderString("name"); static final XContentBuilderString PHYSICAL_NAME = new XContentBuilderString("physical_name"); @@ -484,38 +502,4 @@ public class BlobStoreIndexShardSnapshot { startTime, time, numberOfFiles, totalSize); } - /** - * Returns true if this snapshot contains a file with a given original name - * - * @param physicalName original file name - * @return true if the file was found, false otherwise - */ - public boolean containPhysicalIndexFile(String physicalName) { - return findPhysicalIndexFile(physicalName) != null; - } - - public FileInfo findPhysicalIndexFile(String physicalName) { - for (FileInfo file : indexFiles) { - if (file.physicalName().equals(physicalName)) { - return file; - } - } - return null; - } - - /** - * Returns true if this snapshot contains a file with a given name - * - * @param name file name - * @return true if file was found, false otherwise - */ - public FileInfo findNameFile(String name) { - for (FileInfo file : indexFiles) { - if (file.name().equals(name)) { - return file; - } - } - return null; - } - } diff --git a/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java index a8657655122..11a8689bb26 100644 --- a/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java +++ b/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java @@ -20,10 +20,22 @@ package org.elasticsearch.index.snapshots.blobstore; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; +import java.io.IOException; import java.util.Iterator; import java.util.List; +import java.util.Map; + +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Maps.newHashMap; /** * Contains information about all snapshot for the given shard in repository @@ -31,19 +43,72 @@ import java.util.List; * This class is used to find files that were already snapshoted and clear out files that no longer referenced by any * snapshots */ -public class BlobStoreIndexShardSnapshots implements Iterable { - private final ImmutableList shardSnapshots; +public class BlobStoreIndexShardSnapshots implements Iterable, ToXContent { + private final ImmutableList shardSnapshots; + private final ImmutableMap files; + private final ImmutableMap> physicalFiles; - public BlobStoreIndexShardSnapshots(List shardSnapshots) { + public BlobStoreIndexShardSnapshots(List shardSnapshots) { this.shardSnapshots = ImmutableList.copyOf(shardSnapshots); + // Map between blob names and file info + Map newFiles = newHashMap(); + // Map between original physical names and file info + Map> physicalFiles = newHashMap(); + for (SnapshotFiles snapshot : shardSnapshots) { + // First we build map between filenames in the repo and their original file info + // this map will be used in the next loop + for (FileInfo fileInfo : snapshot.indexFiles()) { + FileInfo oldFile = newFiles.put(fileInfo.name(), fileInfo); + assert oldFile == null || oldFile.isSame(fileInfo); + } + // We are doing it in two loops here so we keep only one copy of the fileInfo per blob + // the first loop de-duplicates fileInfo objects that were loaded from different snapshots but refer to + // the same blob + for (FileInfo fileInfo : snapshot.indexFiles()) { + List physicalFileList = physicalFiles.get(fileInfo.physicalName()); + if (physicalFileList == null) { + physicalFileList = newArrayList(); + physicalFiles.put(fileInfo.physicalName(), physicalFileList); + } + physicalFileList.add(newFiles.get(fileInfo.name())); + } + } + ImmutableMap.Builder> mapBuilder = ImmutableMap.builder(); + for (Map.Entry> entry : physicalFiles.entrySet()) { + mapBuilder.put(entry.getKey(), ImmutableList.copyOf(entry.getValue())); + } + this.physicalFiles = mapBuilder.build(); + this.files = ImmutableMap.copyOf(newFiles); } + private BlobStoreIndexShardSnapshots(ImmutableMap files, ImmutableList shardSnapshots) { + this.shardSnapshots = shardSnapshots; + this.files = files; + Map> physicalFiles = newHashMap(); + for (SnapshotFiles snapshot : shardSnapshots) { + for (FileInfo fileInfo : snapshot.indexFiles()) { + List physicalFileList = physicalFiles.get(fileInfo.physicalName()); + if (physicalFileList == null) { + physicalFileList = newArrayList(); + physicalFiles.put(fileInfo.physicalName(), physicalFileList); + } + physicalFileList.add(files.get(fileInfo.name())); + } + } + ImmutableMap.Builder> mapBuilder = ImmutableMap.builder(); + for (Map.Entry> entry : physicalFiles.entrySet()) { + mapBuilder.put(entry.getKey(), ImmutableList.copyOf(entry.getValue())); + } + this.physicalFiles = mapBuilder.build(); + } + + /** * Returns list of snapshots * * @return list of snapshots */ - public ImmutableList snapshots() { + public ImmutableList snapshots() { return this.shardSnapshots; } @@ -51,16 +116,10 @@ public class BlobStoreIndexShardSnapshots implements Iterable findPhysicalIndexFiles(String physicalName) { + return physicalFiles.get(physicalName); } /** @@ -70,17 +129,166 @@ public class BlobStoreIndexShardSnapshots implements Iterable iterator() { + public Iterator iterator() { return shardSnapshots.iterator(); } + + static final class Fields { + static final XContentBuilderString FILES = new XContentBuilderString("files"); + static final XContentBuilderString SNAPSHOTS = new XContentBuilderString("snapshots"); + } + + static final class ParseFields { + static final ParseField FILES = new ParseField("files"); + static final ParseField SNAPSHOTS = new ParseField("snapshots"); + } + + /** + * Writes index file for the shard in the following format. + * 

+     * {@code
+     * {
+     *     "files": [{
+     *         "name": "__3",
+     *         "physical_name": "_0.si",
+     *         "length": 310,
+     *         "checksum": "1tpsg3p",
+     *         "written_by": "5.1.0",
+     *         "meta_hash": "P9dsFxNMdWNlb......"
+     *     }, {
+     *         "name": "__2",
+     *         "physical_name": "segments_2",
+     *         "length": 150,
+     *         "checksum": "11qjpz6",
+     *         "written_by": "5.1.0",
+     *         "meta_hash": "P9dsFwhzZWdtZ......."
+     *     }, {
+     *         "name": "__1",
+     *         "physical_name": "_0.cfe",
+     *         "length": 363,
+     *         "checksum": "er9r9g",
+     *         "written_by": "5.1.0"
+     *     }, {
+     *         "name": "__0",
+     *         "physical_name": "_0.cfs",
+     *         "length": 3354,
+     *         "checksum": "491liz",
+     *         "written_by": "5.1.0"
+     *     }, {
+     *         "name": "__4",
+     *         "physical_name": "segments_3",
+     *         "length": 150,
+     *         "checksum": "134567",
+     *         "written_by": "5.1.0",
+     *         "meta_hash": "P9dsFwhzZWdtZ......."
+     *     }],
+     *     "snapshots": {
+     *         "snapshot_1": {
+     *             "files": ["__0", "__1", "__2", "__3"]
+     *         },
+     *         "snapshot_2": {
+     *             "files": ["__0", "__1", "__2", "__4"]
+     *         }
+     *     }
+     * }
+     * }
+     *
+ */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + // First we list all blobs with their file infos: + builder.startArray(Fields.FILES); + for (Map.Entry entry : files.entrySet()) { + FileInfo.toXContent(entry.getValue(), builder, params); + } + builder.endArray(); + // Then we list all snapshots with list of all blobs that are used by the snapshot + builder.startObject(Fields.SNAPSHOTS); + for (SnapshotFiles snapshot : shardSnapshots) { + builder.startObject(snapshot.snapshot(), XContentBuilder.FieldCaseConversion.NONE); + builder.startArray(Fields.FILES); + for (FileInfo fileInfo : snapshot.indexFiles()) { + builder.value(fileInfo.name()); + } + builder.endArray(); + builder.endObject(); + } + builder.endObject(); + + builder.endObject(); + return builder; + } + + public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + Map> snapshotsMap = newHashMap(); + ImmutableMap.Builder filesBuilder = ImmutableMap.builder(); + if (token == XContentParser.Token.START_OBJECT) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token != XContentParser.Token.FIELD_NAME) { + throw new ElasticsearchParseException("unexpected token [" + token + "]"); + } + String currentFieldName = parser.currentName(); + token = parser.nextToken(); + if (token == XContentParser.Token.START_ARRAY) { + if (ParseFields.FILES.match(currentFieldName) == false) { + throw new ElasticsearchParseException("unknown array [" + currentFieldName + "]"); + } + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + FileInfo fileInfo = FileInfo.fromXContent(parser); + filesBuilder.put(fileInfo.name(), fileInfo); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (ParseFields.SNAPSHOTS.match(currentFieldName) == false) { + throw new ElasticsearchParseException("unknown object [" + currentFieldName + "]"); + } + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token != XContentParser.Token.FIELD_NAME) { + throw new ElasticsearchParseException("unknown object [" + currentFieldName + "]"); + } + String snapshot = parser.currentName(); + if (parser.nextToken() != XContentParser.Token.START_OBJECT) { + throw new ElasticsearchParseException("unknown object [" + currentFieldName + "]"); + } + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + if (parser.nextToken() == XContentParser.Token.START_ARRAY) { + if (ParseFields.FILES.match(currentFieldName) == false) { + throw new ElasticsearchParseException("unknown array [" + currentFieldName + "]"); + } + List fileNames = newArrayList(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + fileNames.add(parser.text()); + } + snapshotsMap.put(snapshot, fileNames); + } + } + } + } + } else { + throw new ElasticsearchParseException("unexpected token [" + token + "]"); + } + } + } + + ImmutableMap files = filesBuilder.build(); + ImmutableList.Builder snapshots = ImmutableList.builder(); + for (Map.Entry> entry : snapshotsMap.entrySet()) { + ImmutableList.Builder fileInfosBuilder = ImmutableList.builder(); + for (String file : entry.getValue()) { + FileInfo fileInfo = files.get(file); + assert fileInfo != null; + fileInfosBuilder.add(fileInfo); + } + snapshots.add(new SnapshotFiles(entry.getKey(), fileInfosBuilder.build())); + } + return new BlobStoreIndexShardSnapshots(files, snapshots.build()); + } + } diff --git a/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java b/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java new file mode 100644 index 00000000000..0dad85d2d54 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/snapshots/blobstore/SnapshotFiles.java @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.snapshots.blobstore; + +import com.google.common.collect.ImmutableList; +import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; + +import java.util.Map; + +import static com.google.common.collect.Maps.newHashMap; + +/** + * Contains a list of files participating in a snapshot + */ +public class SnapshotFiles { + + private final String snapshot; + + private final ImmutableList indexFiles; + + private Map physicalFiles = null; + + public String snapshot() { + return snapshot; + } + + public SnapshotFiles(String snapshot, ImmutableList indexFiles ) { + this.snapshot = snapshot; + this.indexFiles = indexFiles; + } + + /** + * Returns a list of file in the snapshot + */ + public ImmutableList indexFiles() { + return indexFiles; + } + + /** + * Returns true if this snapshot contains a file with a given original name + * + * @param physicalName original file name + * @return true if the file was found, false otherwise + */ + public boolean containPhysicalIndexFile(String physicalName) { + return findPhysicalIndexFile(physicalName) != null; + } + + /** + * Returns information about a physical file with the given name + * @param physicalName the original file name + * @return information about this file + */ + public FileInfo findPhysicalIndexFile(String physicalName) { + if (physicalFiles == null) { + Map files = newHashMap(); + for(FileInfo fileInfo : indexFiles) { + files.put(fileInfo.physicalName(), fileInfo); + } + this.physicalFiles = files; + } + return physicalFiles.get(physicalName); + } + +} diff --git a/src/main/java/org/elasticsearch/index/store/StoreModule.java b/src/main/java/org/elasticsearch/index/store/StoreModule.java index fe0e314ba16..fccd2de2e43 100644 --- a/src/main/java/org/elasticsearch/index/store/StoreModule.java +++ b/src/main/java/org/elasticsearch/index/store/StoreModule.java @@ -27,8 +27,6 @@ import org.elasticsearch.index.shard.ShardPath; * */ public class StoreModule extends AbstractModule { - - private final ShardLock lock; private final Store.OnClose closeCallback; private final ShardPath path; diff --git a/src/main/java/org/elasticsearch/index/suggest/SuggestShardModule.java b/src/main/java/org/elasticsearch/index/suggest/SuggestShardModule.java deleted file mode 100644 index 9e1d0e1375a..00000000000 --- a/src/main/java/org/elasticsearch/index/suggest/SuggestShardModule.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.suggest; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.index.suggest.stats.ShardSuggestService; - -/** - * - */ -public class SuggestShardModule extends AbstractModule { - @Override - protected void configure() { - bind(ShardSuggestService.class).asEagerSingleton(); - } -} diff --git a/src/main/java/org/elasticsearch/index/termvectors/ShardTermVectorsModule.java b/src/main/java/org/elasticsearch/index/termvectors/ShardTermVectorsModule.java deleted file mode 100644 index 45a7d14b703..00000000000 --- a/src/main/java/org/elasticsearch/index/termvectors/ShardTermVectorsModule.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.termvectors; - -import org.elasticsearch.common.inject.AbstractModule; - -/** - * - */ -public class ShardTermVectorsModule extends AbstractModule { - - @Override - protected void configure() { - bind(ShardTermVectorsService.class).asEagerSingleton(); - } -} diff --git a/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 77832bb052d..6995ecae587 100644 --- a/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -41,7 +41,7 @@ import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -369,7 +369,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent cursor : indexMetaData.mappings().values()) { MappingMetaData mappingMd = cursor.value; String mappingType = mappingMd.type(); - CompressedString mappingSource = mappingMd.source(); + CompressedXContent mappingSource = mappingMd.source(); if (mappingType.equals(MapperService.DEFAULT_MAPPING)) { // we processed _default_ first continue; } @@ -396,7 +396,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent(index, mappingType))) { seenMappings.put(new Tuple<>(index, mappingType), true); } @@ -484,7 +484,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent cursor : aliases) { AliasMetaData aliasMd = cursor.value; String alias = aliasMd.alias(); - CompressedString filter = aliasMd.filter(); + CompressedXContent filter = aliasMd.filter(); try { if (!indexAliasesService.hasAlias(alias)) { if (logger.isDebugEnabled()) { diff --git a/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java index 537392a9c98..6c6e38eb87a 100644 --- a/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java +++ b/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java @@ -381,7 +381,7 @@ public class SyncedFlushService extends AbstractComponent { @Override public void handleException(TransportException exp) { - logger.trace("{} error while performing pre synced flush on [{}], skipping", shardId, exp, shard); + logger.trace("{} error while performing pre synced flush on [{}], skipping", exp, shardId, shard); if (countDown.countDown()) { listener.onResponse(commitIds); } diff --git a/src/main/java/org/elasticsearch/node/Node.java b/src/main/java/org/elasticsearch/node/Node.java index 820c3a84534..355bea50643 100644 --- a/src/main/java/org/elasticsearch/node/Node.java +++ b/src/main/java/org/elasticsearch/node/Node.java @@ -36,7 +36,6 @@ import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleComponent; -import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.inject.Injector; import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.lease.Releasable; @@ -151,7 +150,6 @@ public class Node implements Releasable { // create the environment based on the finalized (processed) view of the settings this.environment = new Environment(this.settings()); - CompressorFactory.configure(settings); final NodeEnvironment nodeEnvironment; try { nodeEnvironment = new NodeEnvironment(this.settings, this.environment); diff --git a/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java b/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java index e8973753b2a..68c2722940e 100644 --- a/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java +++ b/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java @@ -20,9 +20,11 @@ package org.elasticsearch.node.internal; import com.google.common.collect.ImmutableList; +import com.google.common.collect.UnmodifiableIterator; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.Names; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.cli.Terminal; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -41,10 +43,38 @@ public class InternalSettingsPreparer { static final List ALLOWED_SUFFIXES = ImmutableList.of(".yml", ".yaml", ".json", ".properties"); + public static final String SECRET_PROMPT_VALUE = "${prompt::secret}"; + public static final String TEXT_PROMPT_VALUE = "${prompt::text}"; + public static final String IGNORE_SYSTEM_PROPERTIES_SETTING = "config.ignore_system_properties"; + + /** + * Prepares the settings by gathering all elasticsearch system properties, optionally loading the configuration settings, + * and then replacing all property placeholders. This method will not work with settings that have __prompt__ + * as their value unless they have been resolved previously. + * @param pSettings The initial settings to use + * @param loadConfigSettings flag to indicate whether to load settings from the configuration directory/file + * @return the {@link Settings} and {@link Environment} as a {@link Tuple} + */ public static Tuple prepareSettings(Settings pSettings, boolean loadConfigSettings) { + return prepareSettings(pSettings, loadConfigSettings, null); + } + + /** + * Prepares the settings by gathering all elasticsearch system properties, optionally loading the configuration settings, + * and then replacing all property placeholders. If a {@link Terminal} is provided and configuration settings are loaded, + * settings with the __prompt__ value will result in a prompt for the setting to the user. + * @param pSettings The initial settings to use + * @param loadConfigSettings flag to indicate whether to load settings from the configuration directory/file + * @param terminal the Terminal to use for input/output + * @return the {@link Settings} and {@link Environment} as a {@link Tuple} + */ + public static Tuple prepareSettings(Settings pSettings, boolean loadConfigSettings, Terminal terminal) { // ignore this prefixes when getting properties from es. and elasticsearch. String[] ignorePrefixes = new String[]{"es.default.", "elasticsearch.default."}; - boolean useSystemProperties = !pSettings.getAsBoolean("config.ignore_system_properties", false); + // ignore the special prompt placeholders since they have the same format as property placeholders and will be resolved + // as having a default value because of the ':' in the format + String[] ignoredPlaceholders = new String[] { SECRET_PROMPT_VALUE, TEXT_PROMPT_VALUE }; + boolean useSystemProperties = !pSettings.getAsBoolean(IGNORE_SYSTEM_PROPERTIES_SETTING, false); // just create enough settings to build the environment Settings.Builder settingsBuilder = settingsBuilder().put(pSettings); if (useSystemProperties) { @@ -53,7 +83,7 @@ public class InternalSettingsPreparer { .putProperties("elasticsearch.", System.getProperties(), ignorePrefixes) .putProperties("es.", System.getProperties(), ignorePrefixes); } - settingsBuilder.replacePropertyPlaceholders(); + settingsBuilder.replacePropertyPlaceholders(ignoredPlaceholders); Environment environment = new Environment(settingsBuilder.build()); @@ -91,17 +121,17 @@ public class InternalSettingsPreparer { settingsBuilder.putProperties("elasticsearch.", System.getProperties(), ignorePrefixes) .putProperties("es.", System.getProperties(), ignorePrefixes); } - settingsBuilder.replacePropertyPlaceholders(); + settingsBuilder.replacePropertyPlaceholders(ignoredPlaceholders); // allow to force set properties based on configuration of the settings provided for (Map.Entry entry : pSettings.getAsMap().entrySet()) { String setting = entry.getKey(); if (setting.startsWith("force.")) { settingsBuilder.remove(setting); - settingsBuilder.put(setting.substring(".force".length()), entry.getValue()); + settingsBuilder.put(setting.substring("force.".length()), entry.getValue()); } } - settingsBuilder.replacePropertyPlaceholders(); + settingsBuilder.replacePropertyPlaceholders(ignoredPlaceholders); // generate the name if (settingsBuilder.get("name") == null) { @@ -123,7 +153,7 @@ public class InternalSettingsPreparer { settingsBuilder.put(ClusterName.SETTING, ClusterName.DEFAULT.value()); } - Settings v1 = settingsBuilder.build(); + Settings v1 = replacePromptPlaceholders(settingsBuilder.build(), terminal); environment = new Environment(v1); // put back the env settings @@ -135,4 +165,45 @@ public class InternalSettingsPreparer { return new Tuple<>(v1, environment); } + + static Settings replacePromptPlaceholders(Settings settings, Terminal terminal) { + UnmodifiableIterator> iter = settings.getAsMap().entrySet().iterator(); + Settings.Builder builder = Settings.builder(); + + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String value = entry.getValue(); + String key = entry.getKey(); + switch (value) { + case SECRET_PROMPT_VALUE: + String secretValue = promptForValue(key, terminal, true); + if (Strings.hasLength(secretValue)) { + builder.put(key, secretValue); + } + break; + case TEXT_PROMPT_VALUE: + String textValue = promptForValue(key, terminal, false); + if (Strings.hasLength(textValue)) { + builder.put(key, textValue); + } + break; + default: + builder.put(key, value); + break; + } + } + + return builder.build(); + } + + static String promptForValue(String key, Terminal terminal, boolean secret) { + if (terminal == null) { + throw new UnsupportedOperationException("found property [" + key + "] with value [" + (secret ? SECRET_PROMPT_VALUE : TEXT_PROMPT_VALUE) +"]. prompting for property values is only supported when running elasticsearch in the foreground"); + } + + if (secret) { + return new String(terminal.readSecret("Enter value for [%s]: ", key)); + } + return terminal.readText("Enter value for [%s]: ", key); + } } diff --git a/src/main/java/org/elasticsearch/plugins/PluginManager.java b/src/main/java/org/elasticsearch/plugins/PluginManager.java index f8802b658c2..31b235ceda7 100644 --- a/src/main/java/org/elasticsearch/plugins/PluginManager.java +++ b/src/main/java/org/elasticsearch/plugins/PluginManager.java @@ -25,6 +25,7 @@ import com.google.common.collect.Iterators; import org.apache.lucene.util.IOUtils; import org.elasticsearch.*; import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.cli.Terminal; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.http.client.HttpDownloadHelper; import org.elasticsearch.common.io.FileSystemUtils; @@ -338,7 +339,7 @@ public class PluginManager { private static final int EXIT_CODE_ERROR = 70; public static void main(String[] args) { - Tuple initialSettings = InternalSettingsPreparer.prepareSettings(EMPTY_SETTINGS, true); + Tuple initialSettings = InternalSettingsPreparer.prepareSettings(EMPTY_SETTINGS, true, Terminal.DEFAULT); try { Files.createDirectories(initialSettings.v2().pluginsFile()); diff --git a/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 180f6595521..a505b4b5493 100644 --- a/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -24,27 +24,35 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.io.ByteStreams; + import org.apache.lucene.store.RateLimiter; -import org.elasticsearch.ElasticsearchException; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.SnapshotId; -import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.compress.CompressorFactory; +import org.elasticsearch.common.compress.NotXContentException; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.shard.IndexShardException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardRepository; @@ -54,14 +62,21 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositorySettings; import org.elasticsearch.repositories.RepositoryVerificationException; -import org.elasticsearch.snapshots.*; +import org.elasticsearch.snapshots.InvalidSnapshotNameException; +import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotCreationException; +import org.elasticsearch.snapshots.SnapshotException; +import org.elasticsearch.snapshots.SnapshotMissingException; +import org.elasticsearch.snapshots.SnapshotShardFailure; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.NoSuchFileException; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import static com.google.common.collect.Lists.newArrayList; @@ -159,9 +174,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent snapshots) throws IOException { BytesStreamOutput bStream = new BytesStreamOutput(); - StreamOutput stream = bStream; - if (isCompress()) { - stream = CompressorFactory.defaultCompressor().streamOutput(stream); - } + StreamOutput stream = compressIfNeeded(bStream); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream); builder.startObject(); builder.startArray("snapshots"); @@ -608,7 +627,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent snapshots = new ArrayList<>(); - try (XContentParser parser = XContentHelper.createParser(data, 0, data.length)) { + try (XContentParser parser = XContentHelper.createParser(new BytesArray(data))) { if (parser.nextToken() == XContentParser.Token.START_OBJECT) { if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { String currentFieldName = parser.currentName(); diff --git a/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java b/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java index 85b46925b5f..9f09081417a 100644 --- a/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java +++ b/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestBuilderListener; @@ -40,11 +41,14 @@ import static org.elasticsearch.rest.RestStatus.OK; */ public class RestGetRepositoriesAction extends BaseRestHandler { + private final SettingsFilter settingsFilter; + @Inject - public RestGetRepositoriesAction(Settings settings, RestController controller, Client client) { + public RestGetRepositoriesAction(Settings settings, RestController controller, Client client, SettingsFilter settingsFilter) { super(settings, controller, client); controller.registerHandler(GET, "/_snapshot", this); controller.registerHandler(GET, "/_snapshot/{repository}", this); + this.settingsFilter = settingsFilter; } @Override @@ -53,6 +57,7 @@ public class RestGetRepositoriesAction extends BaseRestHandler { GetRepositoriesRequest getRepositoriesRequest = getRepositoryRequest(repositories); getRepositoriesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getRepositoriesRequest.masterNodeTimeout())); getRepositoriesRequest.local(request.paramAsBoolean("local", getRepositoriesRequest.local())); + settingsFilter.addFilterSettingParams(request); client.admin().cluster().getRepositories(getRepositoriesRequest, new RestBuilderListener(channel) { @Override public RestResponse buildResponse(GetRepositoriesResponse response, XContentBuilder builder) throws Exception { diff --git a/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java b/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java index d019e598cac..a23780db62e 100644 --- a/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java +++ b/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java @@ -31,12 +31,20 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.index.VersionType; -import org.elasticsearch.rest.*; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestBuilderListener; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptParameterParser; import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue; +import java.util.HashMap; import java.util.Map; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -68,16 +76,13 @@ public class RestUpdateAction extends BaseRestHandler { scriptParameterParser.parseParams(request); ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue(); if (scriptValue != null) { - updateRequest.script(scriptValue.script(), scriptValue.scriptType()); - } - String scriptLang = scriptParameterParser.lang(); - if (scriptLang != null) { - updateRequest.scriptLang(scriptLang); - } - for (Map.Entry entry : request.params().entrySet()) { - if (entry.getKey().startsWith("sp_")) { - updateRequest.addScriptParam(entry.getKey().substring(3), entry.getValue()); + Map scriptParams = new HashMap<>(); + for (Map.Entry entry : request.params().entrySet()) { + if (entry.getKey().startsWith("sp_")) { + scriptParams.put(entry.getKey().substring(3), entry.getValue()); + } } + updateRequest.script(new Script(scriptValue.script(), scriptValue.scriptType(), scriptParameterParser.lang(), scriptParams)); } String sField = request.param("fields"); if (sField != null) { diff --git a/src/main/java/org/elasticsearch/script/AbstractScriptParser.java b/src/main/java/org/elasticsearch/script/AbstractScriptParser.java new file mode 100644 index 00000000000..f03edce9682 --- /dev/null +++ b/src/main/java/org/elasticsearch/script/AbstractScriptParser.java @@ -0,0 +1,196 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.script.Script.ScriptField; +import org.elasticsearch.script.Script.ScriptParseException; +import org.elasticsearch.script.ScriptService.ScriptType; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; + +public abstract class AbstractScriptParser { + + protected abstract String parseInlineScript(XContentParser parser) throws IOException; + + protected abstract S createScript(String script, ScriptType type, String lang, Map params); + + protected abstract S createSimpleScript(XContentParser parser) throws IOException; + + @Deprecated + protected Map getAdditionalScriptParameters() { + return Collections.emptyMap(); + } + + public S parse(XContentParser parser) throws IOException { + + XContentParser.Token token = parser.currentToken(); + // If the parser hasn't yet been pushed to the first token, do it now + if (token == null) { + token = parser.nextToken(); + } + + if (token == XContentParser.Token.VALUE_STRING) { + return createSimpleScript(parser); + } + if (token != XContentParser.Token.START_OBJECT) { + throw new ScriptParseException("expected a string value or an object, but found [{}] instead", token); + } + + String script = null; + ScriptType type = null; + String lang = getDefaultScriptLang(); + Map params = null; + + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (ScriptType.INLINE.getParseField().match(currentFieldName) || ScriptService.SCRIPT_INLINE.match(currentFieldName)) { + type = ScriptType.INLINE; + script = parseInlineScript(parser); + } else if (ScriptType.FILE.getParseField().match(currentFieldName) || ScriptService.SCRIPT_FILE.match(currentFieldName)) { + type = ScriptType.FILE; + if (token == XContentParser.Token.VALUE_STRING) { + script = parser.text(); + } else { + throw new ScriptParseException("expected a string value for field [{}], but found [{}]", currentFieldName, token); + } + } else if (ScriptType.INDEXED.getParseField().match(currentFieldName) || ScriptService.SCRIPT_ID.match(currentFieldName)) { + type = ScriptType.INDEXED; + if (token == XContentParser.Token.VALUE_STRING) { + script = parser.text(); + } else { + throw new ScriptParseException("expected a string value for field [{}], but found [{}]", currentFieldName, token); + } + } else if (ScriptField.LANG.match(currentFieldName) || ScriptService.SCRIPT_LANG.match(currentFieldName)) { + if (token == XContentParser.Token.VALUE_STRING) { + lang = parser.text(); + } else { + throw new ScriptParseException("expected a string value for field [{}], but found [{}]", currentFieldName, token); + } + } else if (ScriptField.PARAMS.match(currentFieldName)) { + if (token == XContentParser.Token.START_OBJECT) { + params = parser.map(); + } else { + throw new ScriptParseException("expected an object for field [{}], but found [{}]", currentFieldName, token); + } + } else { + // TODO remove this in 2.0 + ScriptType paramScriptType = getAdditionalScriptParameters().get(currentFieldName); + if (paramScriptType != null) { + script = parseInlineScript(parser); + type = paramScriptType; + } else { + throw new ScriptParseException("unexpected field [{}]", currentFieldName); + } + } + } + if (script == null) { + throw new ScriptParseException("expected one of [{}], [{}] or [{}] fields, but found none", ScriptType.INLINE.getParseField() + .getPreferredName(), ScriptType.FILE.getParseField().getPreferredName(), ScriptType.INDEXED.getParseField() + .getPreferredName()); + } + assert type != null : "if script is not null, type should definitely not be null"; + return createScript(script, type, lang, params); + + } + + /** + * @return the default script language for this parser or null + * to use the default set in the ScriptService + */ + protected String getDefaultScriptLang() { + return null; + } + + public S parse(Map config, boolean removeMatchedEntries) { + String script = null; + ScriptType type = null; + String lang = null; + Map params = null; + for (Iterator> itr = config.entrySet().iterator(); itr.hasNext();) { + Entry entry = itr.next(); + String parameterName = entry.getKey(); + Object parameterValue = entry.getValue(); + if (ScriptField.LANG.match(parameterName) || ScriptService.SCRIPT_LANG.match(parameterName)) { + if (parameterValue instanceof String || parameterValue == null) { + lang = (String) parameterValue; + if (removeMatchedEntries) { + itr.remove(); + } + } else { + throw new ScriptParseException("Value must be of type String: [" + parameterName + "]"); + } + } else if (ScriptField.PARAMS.match(parameterName)) { + if (parameterValue instanceof Map || parameterValue == null) { + params = (Map) parameterValue; + if (removeMatchedEntries) { + itr.remove(); + } + } else { + throw new ScriptParseException("Value must be of type String: [" + parameterName + "]"); + } + } else if (ScriptType.INLINE.getParseField().match(parameterName) || ScriptService.SCRIPT_INLINE.match(parameterName)) { + if (parameterValue instanceof String || parameterValue == null) { + script = (String) parameterValue; + type = ScriptType.INLINE; + if (removeMatchedEntries) { + itr.remove(); + } + } else { + throw new ScriptParseException("Value must be of type String: [" + parameterName + "]"); + } + } else if (ScriptType.FILE.getParseField().match(parameterName) || ScriptService.SCRIPT_FILE.match(parameterName)) { + if (parameterValue instanceof String || parameterValue == null) { + script = (String) parameterValue; + type = ScriptType.FILE; + if (removeMatchedEntries) { + itr.remove(); + } + } else { + throw new ScriptParseException("Value must be of type String: [" + parameterName + "]"); + } + } else if (ScriptType.INDEXED.getParseField().match(parameterName) || ScriptService.SCRIPT_ID.match(parameterName)) { + if (parameterValue instanceof String || parameterValue == null) { + script = (String) parameterValue; + type = ScriptType.INDEXED; + if (removeMatchedEntries) { + itr.remove(); + } + } else { + throw new ScriptParseException("Value must be of type String: [" + parameterName + "]"); + } + } + } + if (script == null) { + throw new ScriptParseException("expected one of [{}], [{}] or [{}] fields, but found none", ScriptType.INLINE.getParseField() + .getPreferredName(), ScriptType.FILE.getParseField().getPreferredName(), ScriptType.INDEXED.getParseField() + .getPreferredName()); + } + assert type != null : "if script is not null, type should definitely not be null"; + return createScript(script, type, lang, params); + } + +} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/script/Script.java b/src/main/java/org/elasticsearch/script/Script.java index 655ff82c08e..d826eaad8ed 100644 --- a/src/main/java/org/elasticsearch/script/Script.java +++ b/src/main/java/org/elasticsearch/script/Script.java @@ -19,52 +19,93 @@ package org.elasticsearch.script; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.logging.support.LoggerMessageFormat; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.script.ScriptService.ScriptType; + +import java.io.IOException; import java.util.Map; - -import static org.elasticsearch.script.ScriptService.ScriptType; - /** * Script holds all the parameters necessary to compile or find in cache and then execute a script. */ -public class Script { +public class Script implements ToXContent, Streamable { - private final String lang; - private final String script; - private final ScriptType type; - private final Map params; + public static final ScriptType DEFAULT_TYPE = ScriptType.INLINE; + private static final ScriptParser PARSER = new ScriptParser(); + + private String script; + private @Nullable ScriptType type; + private @Nullable String lang; + private @Nullable Map params; + + /** + * For Serialization + */ + Script() { + } + + /** + * Constructor for simple inline script. The script will have no lang or + * params set. + * + * @param script + * The inline script to execute. + */ + public Script(String script) { + if (script == null) { + throw new IllegalArgumentException("The parameter script (String) must not be null in Script."); + } + this.script = script; + } + + /** + * For sub-classes to use to override the default language + */ + protected Script(String script, String lang) { + if (script == null) { + throw new IllegalArgumentException("The parameter script (String) must not be null in Script."); + } + this.script = script; + this.lang = lang; + } /** * Constructor for Script. - * @param lang The language of the script to be compiled/executed. - * @param script The cache key of the script to be compiled/executed. For dynamic scripts this is the actual - * script source code. For indexed scripts this is the id used in the request. For on disk scripts - * this is the file name. - * @param type The type of script -- dynamic, indexed, or file. - * @param params The map of parameters the script will be executed with. + * + * @param script + * The cache key of the script to be compiled/executed. For + * inline scripts this is the actual script source code. For + * indexed scripts this is the id used in the request. For on + * file scripts this is the file name. + * @param type + * The type of script -- dynamic, indexed, or file. + * @param lang + * The language of the script to be compiled/executed. + * @param params + * The map of parameters the script will be executed with. */ - public Script(String lang, String script, ScriptType type, Map params) { + public Script(String script, ScriptType type, @Nullable String lang, @Nullable Map params) { if (script == null) { throw new IllegalArgumentException("The parameter script (String) must not be null in Script."); } if (type == null) { throw new IllegalArgumentException("The parameter type (ScriptType) must not be null in Script."); } - - this.lang = lang; this.script = script; this.type = type; + this.lang = lang; this.params = params; } - /** - * Method for getting language. - * @return The language of the script to be compiled/executed. - */ - public String getLang() { - return lang; - } - /** * Method for getting the script. * @return The cache key of the script to be compiled/executed. For dynamic scripts this is the actual @@ -77,17 +118,190 @@ public class Script { /** * Method for getting the type. - * @return The type of script -- dynamic, indexed, or file. + * + * @return The type of script -- inline, indexed, or file. */ public ScriptType getType() { - return type; + return type == null ? DEFAULT_TYPE : type; + } + + /** + * Method for getting language. + * + * @return The language of the script to be compiled/executed. + */ + public String getLang() { + return lang; } /** * Method for getting the parameters. + * * @return The map of parameters the script will be executed with. */ public Map getParams() { return params; } + + @Override + public final void readFrom(StreamInput in) throws IOException { + script = in.readString(); + if (in.readBoolean()) { + type = ScriptType.readFrom(in); + } + lang = in.readOptionalString(); + if (in.readBoolean()) { + params = in.readMap(); + } + doReadFrom(in); + } + + protected void doReadFrom(StreamInput in) throws IOException { + // For sub-classes to Override + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + out.writeString(script); + boolean hasType = type != null; + out.writeBoolean(hasType); + if (hasType) { + ScriptType.writeTo(type, out); + } + out.writeOptionalString(lang); + boolean hasParams = params != null; + out.writeBoolean(hasParams); + if (hasParams) { + out.writeMap(params); + } + doWriteTo(out); + } + + protected void doWriteTo(StreamOutput out) throws IOException { + // For sub-classes to Override + } + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params builderParams) throws IOException { + if (type == null) { + return builder.value(script); + } + + builder.startObject(); + scriptFieldToXContent(script, type, builder, builderParams); + if (lang != null) { + builder.field(ScriptField.LANG.getPreferredName(), lang); + } + if (params != null) { + builder.field(ScriptField.PARAMS.getPreferredName(), params); + } + builder.endObject(); + return builder; + } + + protected XContentBuilder scriptFieldToXContent(String script, ScriptType type, XContentBuilder builder, Params builderParams) + throws IOException { + builder.field(type.getParseField().getPreferredName(), script); + return builder; + } + + public static Script readScript(StreamInput in) throws IOException { + Script script = new Script(); + script.readFrom(in); + return script; + } + + public static Script parse(Map config, boolean removeMatchedEntries) { + return PARSER.parse(config, removeMatchedEntries); + } + + public static Script parse(XContentParser parser) throws IOException { + return PARSER.parse(parser); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((lang == null) ? 0 : lang.hashCode()); + result = prime * result + ((params == null) ? 0 : params.hashCode()); + result = prime * result + ((script == null) ? 0 : script.hashCode()); + result = prime * result + ((type == null) ? 0 : type.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Script other = (Script) obj; + if (lang == null) { + if (other.lang != null) + return false; + } else if (!lang.equals(other.lang)) + return false; + if (params == null) { + if (other.params != null) + return false; + } else if (!params.equals(other.params)) + return false; + if (script == null) { + if (other.script != null) + return false; + } else if (!script.equals(other.script)) + return false; + if (type != other.type) + return false; + return true; + } + + @Override + public String toString() { + return "[script: " + script + ", type: " + type.getParseField().getPreferredName() + ", lang: " + lang + ", params: " + params + + "]"; + } + + private static class ScriptParser extends AbstractScriptParser